diff --git a/.env b/.env index 7d3f1201..189e0002 100644 --- a/.env +++ b/.env @@ -177,3 +177,21 @@ REGISTRY_VERSION=2.8.3 REGISTRY_HOST=registry REGISTRY_PORT=5000 REGISTRY_EXTERNAL_PORT=5001 + +# milvus +MILVUS_IMAGE=milvusdb/milvus +MILVUS_VERSION=v2.4.1 +MILVUS_HOST=milvus +MILVUS_DATA_DIR=/var/lib/milvus/etcd +MILVUS_PORT=19530 +MILVUS_METRIC_PORT=9091 + +# minio +MINIO_IMAGE=quay.io/minio/minio +MINIO_VERSION=RELEASE.2024-05-10T01-41-38Z +MINIO_HOST=minio +MINIO_PORT=9000 +MINIO_EXTERNAL_PORT=19000 +MINIO_CONSOLE_PORT=9001 +MINIO_CONSOLE_EXTERNAL_PORT=19001 +MINIO_DATA_DIR=~/.local/instill/minio/data diff --git a/.github/workflows/helm-integration-test-backend.yml b/.github/workflows/helm-integration-test-backend.yml index 3adc261f..f2d992eb 100644 --- a/.github/workflows/helm-integration-test-backend.yml +++ b/.github/workflows/helm-integration-test-backend.yml @@ -79,6 +79,9 @@ jobs: --set rayService.workerGroupSpecs[0].resources.limits.memory=2Gi \ --set rayService.workerGroupSpecs[0].resources.requests.cpu=1 \ --set rayService.workerGroupSpecs[0].resources.requests.memory=2Gi \ + --set milvus.broker.resources.requests.cpu=0.25 \ + --set milvus.autorecovery.resources.requests.cpu=0.25 \ + --set milvus.proxy.resources.requests.cpu=0.25 \ --set tags.observability=false \ --set tags.prometheusStack=false @@ -189,6 +192,9 @@ jobs: --set rayService.workerGroupSpecs[0].resources.limits.memory=2Gi \ --set rayService.workerGroupSpecs[0].resources.requests.cpu=1 \ --set rayService.workerGroupSpecs[0].resources.requests.memory=2Gi \ + --set milvus.broker.resources.requests.cpu=0.25 \ + --set milvus.autorecovery.resources.requests.cpu=0.25 \ + --set milvus.proxy.resources.requests.cpu=0.25 \ --set tags.observability=false \ --set tags.prometheusStack=false diff --git a/.github/workflows/helm-integration-test-console.yml b/.github/workflows/helm-integration-test-console.yml index a5574a87..e2ecfc76 100644 --- a/.github/workflows/helm-integration-test-console.yml +++ b/.github/workflows/helm-integration-test-console.yml @@ -68,6 +68,9 @@ jobs: --set rayService.workerGroupSpecs[0].resources.limits.memory=2Gi \ --set rayService.workerGroupSpecs[0].resources.requests.cpu=1 \ --set rayService.workerGroupSpecs[0].resources.requests.memory=2Gi \ + --set milvus.broker.resources.requests.cpu=0.25 \ + --set milvus.autorecovery.resources.requests.cpu=0.25 \ + --set milvus.proxy.resources.requests.cpu=0.25 \ --set tags.observability=false \ --set tags.prometheusStack=false @@ -201,6 +204,9 @@ jobs: --set rayService.workerGroupSpecs[0].resources.limits.memory=2Gi \ --set rayService.workerGroupSpecs[0].resources.requests.cpu=1 \ --set rayService.workerGroupSpecs[0].resources.requests.memory=2Gi \ + --set milvus.broker.resources.requests.cpu=0.25 \ + --set milvus.autorecovery.resources.requests.cpu=0.25 \ + --set milvus.proxy.resources.requests.cpu=0.25 \ --set tags.observability=false \ --set tags.prometheusStack=false diff --git a/charts/core/Chart.lock b/charts/core/Chart.lock index d5a9ed46..1b39de94 100644 --- a/charts/core/Chart.lock +++ b/charts/core/Chart.lock @@ -1,4 +1,7 @@ dependencies: +- name: milvus + repository: https://zilliztech.github.io/milvus-helm/ + version: 4.1.30 - name: kuberay-operator repository: https://ray-project.github.io/kuberay-helm/ version: 1.1.1 @@ -20,5 +23,5 @@ dependencies: - name: grafana repository: https://grafana.github.io/helm-charts version: 7.0.19 -digest: sha256:54af8013247538db390dec1acf2dfb3b7b0d70c039e1d57763d689b6f48aea8e -generated: "2024-05-12T04:49:12.15233+08:00" +digest: sha256:12d974d5198de61d676406882a96f0d7e589bc4eef28e2ae4c303a5b1d787b26 +generated: "2024-05-21T00:12:29.456857+08:00" diff --git a/charts/core/Chart.yaml b/charts/core/Chart.yaml index 04e86b1d..9ab0faf9 100644 --- a/charts/core/Chart.yaml +++ b/charts/core/Chart.yaml @@ -18,6 +18,9 @@ maintainers: - name: Instill AI LTD url: https://github.com/instill-ai dependencies: + - name: milvus + repository: https://zilliztech.github.io/milvus-helm/ + version: 4.1.30 - name: kuberay-operator repository: https://ray-project.github.io/kuberay-helm/ version: 1.1.1 diff --git a/charts/core/README.md b/charts/core/README.md index 6fa1a205..07e525ec 100644 --- a/charts/core/README.md +++ b/charts/core/README.md @@ -8,14 +8,14 @@ The Helm chart of Instill Core | Repository | Name | Version | |------------|------|---------| -| https://charts.bitnami.com/bitnami | etcd | 8.8.1 | | https://grafana.github.io/helm-charts | grafana | 7.0.19 | | https://helm.elastic.co | elasticsearch | 7.17.3 | | https://helm.influxdata.com | influxdb2 | 2.1.1 | | https://jaegertracing.github.io/helm-charts | jaeger | 0.71.2 | | https://open-telemetry.github.io/opentelemetry-helm-charts | opentelemetry-collector | 0.59.1 | | https://prometheus-community.github.io/helm-charts | kube-prometheus-stack | 48.2.1 | -| https://ray-project.github.io/kuberay-helm/ | kuberay-operator | 1.0.0 | +| https://ray-project.github.io/kuberay-helm | kuberay-operator | 1.1.1 | +| https://zilliztech.github.io/milvus-helm | milvus | 4.1.30 | ## Install diff --git a/charts/core/charts/milvus-4.1.30.tgz b/charts/core/charts/milvus-4.1.30.tgz new file mode 100644 index 00000000..b9e0740d Binary files /dev/null and b/charts/core/charts/milvus-4.1.30.tgz differ diff --git a/charts/core/templates/_helpers.tpl b/charts/core/templates/_helpers.tpl index e61ec45d..931658c2 100644 --- a/charts/core/templates/_helpers.tpl +++ b/charts/core/templates/_helpers.tpl @@ -389,6 +389,21 @@ registry {{- printf "5001" -}} {{- end -}} +{{/* +milvus +*/}} +{{- define "core.milvus" -}} + {{- printf "%s-milvus" (include "core.fullname" .) -}} +{{- end -}} + +{{- define "core.milvus.port" -}} + {{- printf "19530" -}} +{{- end -}} + +{{- define "core.milvus.metricPort" -}} + {{- printf "9091" -}} +{{- end -}} + {{/* internal TLS secret names */}} diff --git a/charts/core/values.yaml b/charts/core/values.yaml index 5a5c4946..0b1fba7f 100644 --- a/charts/core/values.yaml +++ b/charts/core/values.yaml @@ -674,11 +674,11 @@ rayService: headGroupSpec: resources: limits: - cpu: "2" + cpu: "1" memory: "4Gi" nvidia.com/gpu: 0 requests: - cpu: "2" + cpu: "1" memory: "4Gi" nvidia.com/gpu: 0 affinity: {} @@ -1471,6 +1471,97 @@ registry: idletimeout: 300s tls: enabled: false +# -- The configuration of milvus +# https://github.com/zilliztech/milvus-helm/blob/master/charts/milvus/values.yaml +milvus: + image: + all: + repository: milvusdb/milvus + tag: v2.4.1 + pullPolicy: IfNotPresent + nodeselector: {} + tolerations: [] + affinity: {} + log: + level: "info" + file: + maxSize: 10 # MB + maxAge: 12 # day + maxBackups: 2 + format: "json" # text/json + cluster: + enabled: true + rootCoordinator: + enabled: false + queryCoordinator: + enabled: false + indexCoordinator: + enabled: false + dataCoordinator: + enabled: false + mixCoordinator: + enabled: true + minio: + mode: standalone + replicas: 1 + persistence: + size: 50Gi + etcd: + enabled: true + replicaCount: 1 + pulsar: + enabled: false + bookkeeper: + replicaCount: 1 + volumes: + journal: + name: journal + size: 10Gi + ledgers: + name: ledgers + size: 20Gi + resources: + requests: + memory: + cpu: 0.5 + kafka: + enabled: true + replicaCount: 1 + defaultReplicationFactor: 1 + offsetsTopicReplicationFactor: 1 + persistence: + enabled: true + storageClass: + accessMode: ReadWriteOnce + size: 30Gi + zookeeper: + enabled: true + replicaCount: 1 + broker: + replicaCount: 1 + resources: + requests: + cpu: 0.5 + autorecovery: + resources: + requests: + cpu: 0.5 + proxy: + replicaCount: 1 + resources: + requests: + cpu: 0.5 + # https://github.com/milvus-io/milvus/blob/master/configs/milvus.yaml + extraConfigFiles: + user.yaml: |+ + etcd: + rootPath: core-milvus + log: + level: info + minio: + bucketName: core-milvus + pulsar: + namespace: instill-ai tags: model: true observability: true diff --git a/docker-compose-latest.yml b/docker-compose-latest.yml index f48adae2..77f07dbf 100644 --- a/docker-compose-latest.yml +++ b/docker-compose-latest.yml @@ -166,3 +166,13 @@ services: registry: ports: - ${REGISTRY_EXTERNAL_PORT}:5000 + + milvus: + ports: + - ${MILVUS_PORT}:19530 + command: milvus run standalone + + minio: + ports: + - ${MINIO_EXTERNAL_PORT}:${MINIO_PORT} + - ${MINIO_CONSOLE_EXTERNAL_PORT}:${MINIO_CONSOLE_PORT} diff --git a/docker-compose.yml b/docker-compose.yml index 539d4dc9..966b1cae 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -50,10 +50,9 @@ services: "curl -f http://${API_GATEWAY_HOST}:${API_GATEWAY_PORT}/__health", ] start_period: 20s - start_interval: 1s - interval: 1s + interval: 30s timeout: 5s - retries: 10 + retries: 2 depends_on: influxdb: condition: service_healthy @@ -93,10 +92,9 @@ services: "wget --no-verbose --tries=1 --spider http://${MGMT_BACKEND_HOST}:${MGMT_BACKEND_PUBLICPORT}/v1beta/health/mgmt", ] start_period: 20s - start_interval: 1s - interval: 1s + interval: 30s timeout: 5s - retries: 10 + retries: 2 depends_on: pg_sql: condition: service_healthy @@ -170,10 +168,9 @@ services: "curl -f http://${PIPELINE_BACKEND_HOST}:${PIPELINE_BACKEND_PUBLICPORT}/v1beta/health/pipeline", ] start_period: 20s - start_interval: 1s - interval: 1s + interval: 30s timeout: 5s - retries: 10 + retries: 2 depends_on: temporal: condition: service_healthy @@ -253,10 +250,9 @@ services: "curl -f http://${MODEL_BACKEND_HOST}:${MODEL_BACKEND_PUBLICPORT}/v1alpha/health/model", ] start_period: 20s - start_interval: 1s - interval: 1s + interval: 30s timeout: 5s - retries: 10 + retries: 2 depends_on: model_backend_worker: condition: service_started @@ -338,10 +334,9 @@ services: "wget --no-verbose --tries=1 --spider http://${ARTIFACT_BACKEND_HOST}:${ARTIFACT_BACKEND_PUBLICPORT}/v1alpha/health/artifact", ] start_period: 20s - start_interval: 1s - interval: 1s + interval: 30s timeout: 5s - retries: 10 + retries: 2 depends_on: pg_sql: condition: service_healthy @@ -386,10 +381,9 @@ services: healthcheck: test: ["CMD-SHELL", "tctl --address temporal:7233 cluster health"] start_period: 20s - start_interval: 1s - interval: 1s + interval: 30s timeout: 5s - retries: 10 + retries: 2 depends_on: pg_sql: condition: service_healthy @@ -436,7 +430,7 @@ services: - ${CONFIG_DIR_PATH}/registry/registries.conf:/etc/containers/registries.conf healthcheck: test: ["CMD", "ray", "status"] - start_period: 45s + start_period: 20s interval: 60s timeout: 5s retries: 2 @@ -469,10 +463,9 @@ services: healthcheck: test: ["CMD-SHELL", "redis-cli --raw incr ping"] start_period: 20s - start_interval: 1s - interval: 1s + interval: 30s timeout: 5s - retries: 10 + retries: 2 elasticsearch: container_name: ${ELASTICSEARCH_HOST} @@ -495,10 +488,9 @@ services: "curl -f http://${ELASTICSEARCH_HOST}:9200/_cluster/health", ] start_period: 20s - start_interval: 1s - interval: 1s + interval: 30s timeout: 5s - retries: 10 + retries: 2 influxdb: container_name: ${INFLUXDB_HOST} @@ -520,10 +512,9 @@ services: test: ["CMD-SHELL", "curl -f http://${INFLUXDB_HOST}:${INFLUXDB_PORT}/health"] start_period: 20s - start_interval: 1s - interval: 1s + interval: 30s timeout: 5s - retries: 10 + retries: 2 redoc_openapi: container_name: redoc-openapi @@ -565,3 +556,33 @@ services: depends_on: redis: condition: service_healthy + + milvus: + container_name: ${MILVUS_HOST} + image: ${MILVUS_IMAGE}:${MILVUS_VERSION} + restart: unless-stopped + environment: + ETCD_USE_EMBED: true + COMMON_STORAGETYPE: local + command: milvus run standalone 1> /dev/null + healthcheck: + test: + ["CMD-SHELL", "curl -f http://${MILVUS_HOST}:${MILVUS_METRIC_PORT}/healthz"] + start_period: 20s + interval: 30s + timeout: 5s + retries: 3 + + minio: + container_name: ${MINIO_HOST} + image: ${MINIO_IMAGE}:${MINIO_VERSION} + restart: unless-stopped + environment: + MINIO_CONSOLE_PORT: ${MINIO_CONSOLE_PORT} + volumes: + - ${MINIO_DATA_DIR}:/data + command: + - server + - /data + - --console-address + - ":${MINIO_CONSOLE_PORT}"