From 85bbb9466aa6a85d5fd411fd9b97965742394f82 Mon Sep 17 00:00:00 2001 From: Joseph Soto Date: Thu, 21 Nov 2024 17:42:25 -0700 Subject: [PATCH] [teraslice] Add Teraslice helm chart (#3834) This PR makes the following changes: - Adds `teraslice` helm chart - Adds helmfile with basic configuration - This allows users to quickly try the helm chart with `opensearch1` and start a simple job Ref to issue ##3828 --------- Co-authored-by: Austin Godber --- examples/helm/README.md | 135 ++++++++++ examples/helm/default-values.yaml | 10 + examples/helm/helmfile.yaml | 49 ++++ examples/helm/kindConfig.yaml | 10 + examples/helm/templates/os1.yaml.gotmpl | 36 +++ helm/teraslice/Chart.yaml | 11 + helm/teraslice/README.md | 4 + helm/teraslice/templates/_helpers.tpl | 197 +++++++++++++++ .../teraslice/templates/configmap-master.yaml | 13 + .../teraslice/templates/configmap-worker.yaml | 13 + helm/teraslice/templates/deployment.yaml | 144 +++++++++++ helm/teraslice/templates/ingress.yaml | 61 +++++ helm/teraslice/templates/persistence.yaml | 25 ++ helm/teraslice/templates/podmonitor.yaml | 46 ++++ helm/teraslice/templates/priorityclass.yaml | 16 ++ helm/teraslice/templates/prometheusrules.yaml | 13 + helm/teraslice/templates/role.yaml | 11 + helm/teraslice/templates/rolebinding.yaml | 15 ++ helm/teraslice/templates/service.yaml | 57 +++++ helm/teraslice/templates/serviceaccount.yaml | 12 + helm/teraslice/templates/servicemonitor.yaml | 30 +++ .../templates/tests/test-connection.yaml | 16 ++ helm/teraslice/values.yaml | 239 ++++++++++++++++++ 23 files changed, 1163 insertions(+) create mode 100644 examples/helm/README.md create mode 100644 examples/helm/default-values.yaml create mode 100644 examples/helm/helmfile.yaml create mode 100644 examples/helm/kindConfig.yaml create mode 100644 examples/helm/templates/os1.yaml.gotmpl create mode 100644 helm/teraslice/Chart.yaml create mode 100644 helm/teraslice/README.md create mode 100644 helm/teraslice/templates/_helpers.tpl create mode 100644 helm/teraslice/templates/configmap-master.yaml create mode 100644 helm/teraslice/templates/configmap-worker.yaml create mode 100644 helm/teraslice/templates/deployment.yaml create mode 100644 helm/teraslice/templates/ingress.yaml create mode 100644 helm/teraslice/templates/persistence.yaml create mode 100644 helm/teraslice/templates/podmonitor.yaml create mode 100644 helm/teraslice/templates/priorityclass.yaml create mode 100644 helm/teraslice/templates/prometheusrules.yaml create mode 100644 helm/teraslice/templates/role.yaml create mode 100644 helm/teraslice/templates/rolebinding.yaml create mode 100644 helm/teraslice/templates/service.yaml create mode 100644 helm/teraslice/templates/serviceaccount.yaml create mode 100644 helm/teraslice/templates/servicemonitor.yaml create mode 100644 helm/teraslice/templates/tests/test-connection.yaml create mode 100644 helm/teraslice/values.yaml diff --git a/examples/helm/README.md b/examples/helm/README.md new file mode 100644 index 00000000000..bbdb56714cd --- /dev/null +++ b/examples/helm/README.md @@ -0,0 +1,135 @@ +# Starting teraslice using helm and helmfile + +## Prerequisites + +The following dependencies are required to successfully deploy a basic instance of Teraslice and interact with its API. The examples provided use Homebrew (brew) and Node.js's (npm) for installation. + +- Docker +- [Helm](https://helm.sh/docs/intro/install/) + - `brew install helm` +- [helmfile](https://formulae.brew.sh/formula/helmfile) + - `brew install helmfile` +- [Kubectl](https://kubernetes.io/docs/reference/kubectl/) + - `brew install kubectl` +- [Kind](https://kind.sigs.k8s.io/) - Kubernetes in Docker + - `brew install kind` +- [curl](https://formulae.brew.sh/formula/curl) - Command-line tool for making HTTP requests + - `brew install curl` +- [teraslice-cli](https://www.npmjs.com/package/teraslice-cli) - A CLI tool for managing Teraslice + - `npm install -g teraslice-cli` + +### Initial Setup + +First you're going to want to be in the correct directory. Starting in the top level of the teraslice directory: + +```bash +cd ./examples/helm +``` + +### Step 1: Creating a Kind Cluster + +Create a single node Kubernetes cluster by running the following command: + +```bash +kind create cluster --config kindConfig.yaml +``` + +### Step 2: Building the Teraslice Docker Image + +Build the teraslice docker image using the following command. Ensure the image is tagged correctly to match the intended version `dev-nodev22.9.0` in this example: + +```bash +docker build -t ghcr.io/terascope/teraslice:dev-nodev22.9.0 ../../. +``` + +### Step 3: Loading the Teraslice Docker Image into the Kind Cluster + +Load the Teraslice Docker image, built above, into the Kind cluster's control plane: + +```bash +kind load docker-image --name k8s-env ghcr.io/terascope/teraslice:dev-nodev22.9.0 +``` + +### Step 4: Verifying the Image Load + +Confirm that the teraslice image has been successfully loaded into the cluster. The following command lists the images available in the clusters control plane: + +```bash +docker exec -it k8s-env-control-plane crictl images +``` + +### Step 5: Verifying the Kubernetes Resource Configuration + +Generate a preview of the Kubernetes resources that will be deployed. This step ensures that the `helmfile` is configured correctly: + +```bash +helmfile diff +``` + +### Step 6: Deploying Resources + +Lastly if there were no errors with the `diff` command, deploy teraslice and opensearch into the cluster by running: + +```bash +helmfile sync +``` + +### Step 7: Deploying Assets + +The example job requires the `standard-assets` and `elasticsearch-assets` to be available in the cluster for successful execution. Use the `teraslice-cli` tool to deploy these assets: + +```bash +teraslice-cli assets deploy localhost terascope/standard-assets +``` + +```bash +teraslice-cli assets deploy localhost terascope/elasticsearch-assets +``` + +### Step 8: Submitting and Starting a Test Job + +This example job generates `10,000` records and writes them to an Opensearch index named `random-data-1`. Submit the job to the Teraslice API using the following command: + +```bash +curl -XPOST 'localhost:5678/v1/jobs' -H "Content-Type: application/json" -d '{ + "name": "data-to-es", + "lifecycle": "once", + "workers": 1, + "assets": [ + "standard", + "elasticsearch" + ], + "operations": [ + { + "_op": "data_generator", + "size": 10000 + }, + { + "_op": "elasticsearch_bulk", + "size": 10000, + "index": "random-data-1" + } + ] +}' + +``` + +### Step 9: Viewing results in opensearch + +Once the job completes, query Opensearch to verify that the documents have been written successfully. Use the following command to view the index information: + +```bash +curl 'localhost:9200/_cat/indices?v&h=index,status,docs.count,docs.deleted,store.size,pri.store.size' +``` + +Results: + +```bash +index status docs.count docs.deleted store.size pri.store.size +teraslice__assets open 2 0 2.8mb 2.8mb +teraslice__state-2024.11 open 1 0 28.8kb 28.8kb +teraslice__ex open 1 0 49.1kb 49.1kb +teraslice__jobs open 1 0 5.6kb 5.6kb +random-data-1 open 10000 0 7mb 7mb +teraslice__analytics-2024.11 open 4 0 23.9kb 23.9kb +``` diff --git a/examples/helm/default-values.yaml b/examples/helm/default-values.yaml new file mode 100644 index 00000000000..fc35c960587 --- /dev/null +++ b/examples/helm/default-values.yaml @@ -0,0 +1,10 @@ +opensearch1: + # If false, opensearch1 will be excluded on helmfile sync + enabled: true + version: 1.3.14 + esJavaOpts: -Xmx512M -Xms512M + memoryLimit: 100Mi + persistentVolumeSize: 8Gi + # The number of replicas + instances: 1 + diff --git a/examples/helm/helmfile.yaml b/examples/helm/helmfile.yaml new file mode 100644 index 00000000000..a9d85ad35d8 --- /dev/null +++ b/examples/helm/helmfile.yaml @@ -0,0 +1,49 @@ +environments: + default: + values: + - default-values.yaml +--- + +repositories: + - name: opensearch + url: https://opensearch-project.github.io/helm-charts/ + +helmDefaults: + wait: true + +releases: + - name: opensearch1 + namespace: ts-dev1 + version: 2.17.1 + chart: opensearch/opensearch + values: + - ./templates/os1.yaml.gotmpl + + - name: teraslice + namespace: ts-dev1 + version: 0.8.12 + chart: ../../helm/teraslice/ + needs: + - ts-dev1/{{ .Values | get "ts-dev1.stateCluster" "opensearch1" }} + values: + - terafoundation: + connectors: + elasticsearch-next: + default: + node: + - "http://opensearch1.ts-dev1:9200" + service: + nodePort: 30678 + type: NodePort + master: + teraslice: + kubernetes_namespace: ts-dev1 + cluster_manager_type: kubernetesV2 + asset_storage_connection_type: elasticsearch-next + worker: + teraslice: + kubernetes_namespace: ts-dev1 + cluster_manager_type: kubernetesV2 + asset_storage_connection_type: elasticsearch-next + image: + tag: dev-nodev22.9.0 diff --git a/examples/helm/kindConfig.yaml b/examples/helm/kindConfig.yaml new file mode 100644 index 00000000000..ace1176da79 --- /dev/null +++ b/examples/helm/kindConfig.yaml @@ -0,0 +1,10 @@ +kind: Cluster +name: k8s-env +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: +- role: control-plane + extraPortMappings: + - containerPort: 30678 # Map internal teraslice api service to host port + hostPort: 5678 + - containerPort: 30921 # Map internal opensearch1 service to host port + hostPort: 9200 diff --git a/examples/helm/templates/os1.yaml.gotmpl b/examples/helm/templates/os1.yaml.gotmpl new file mode 100644 index 00000000000..bd217d5d40a --- /dev/null +++ b/examples/helm/templates/os1.yaml.gotmpl @@ -0,0 +1,36 @@ +replicas: {{ .Values | get "opensearch1.instances" 1 }} + +{{- if eq (.Values | get "opensearch1.instances" 1) 1 }} +singleNode: true +{{- else }} +singleNode: false +{{- end }} + +image: + tag: {{ .Values | get "opensearch1.version" "1.3.14" }} + +service: + type: NodePort + port: 9200 + nodePort: 30921 + +config: + opensearch.yml: + plugins: + security: + disabled: true + {{- if eq (.Values | get "opensearch1.instances" 1) 1 }} + discovery.type: single-node + {{- end }} + +clusterName: opensearch1-cluster + +masterService: "opensearch1" + +resources: + requests: + cpu: "1000m" + memory: {{ .Values | get "opensearch1.memoryLimit" "100Mi" }} + +persistence: + size: {{ .Values | get "opensearch1.persistentVolumeSize" "8Gi" }} diff --git a/helm/teraslice/Chart.yaml b/helm/teraslice/Chart.yaml new file mode 100644 index 00000000000..df25ed2dcde --- /dev/null +++ b/helm/teraslice/Chart.yaml @@ -0,0 +1,11 @@ +apiVersion: v2 +name: teraslice +description: Teraslice - Distributed computing platform for processing JSON data +type: application +version: 0.8.12 +appVersion: v2.7.0 +sources: + - https://github.com/terascope/teraslice +keywords: + - teraslice + - elasticsearch diff --git a/helm/teraslice/README.md b/helm/teraslice/README.md new file mode 100644 index 00000000000..84349ea0c7b --- /dev/null +++ b/helm/teraslice/README.md @@ -0,0 +1,4 @@ + +# Teraslice Helm Chart + +https://terascope.github.io/teraslice/docs/configuration/clustering-k8s diff --git a/helm/teraslice/templates/_helpers.tpl b/helm/teraslice/templates/_helpers.tpl new file mode 100644 index 00000000000..37866fa0598 --- /dev/null +++ b/helm/teraslice/templates/_helpers.tpl @@ -0,0 +1,197 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "teraslice.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "teraslice.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "teraslice.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Common labels +*/}} +{{- define "teraslice.labels" -}} +helm.sh/chart: {{ include "teraslice.chart" . }} +{{ include "teraslice.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} + +{{/* +Selector labels +*/}} +{{- define "teraslice.selectorLabels" -}} +app.kubernetes.io/name: {{ include "teraslice.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} + +{{/* +Master common labels +*/}} +{{- define "teraslice.master.labels" -}} +{{ include "teraslice.labels" . }} +app.kubernetes.io/component: master +{{- end -}} + +{{/* +Master selector labels +*/}} +{{- define "teraslice.master.selectorLabels" -}} +{{ include "teraslice.selectorLabels" . }} +app.kubernetes.io/component: master +{{- end -}} + +{{/* +Worker common labels +*/}} +{{- define "teraslice.worker.labels" -}} +{{ include "teraslice.labels" . }} +app.kubernetes.io/component: worker +{{- end -}} + +{{/* +Worker selector labels +*/}} +{{- define "teraslice.worker.selectorLabels" -}} +{{ include "teraslice.selectorLabels" . }} +app.kubernetes.io/component: worker +{{- end -}} + +{{/* +Create the name of the service account to use +*/}} +{{- define "teraslice.serviceAccountName" -}} +{{- if .Values.serviceAccount.create -}} + {{ default (include "teraslice.fullname" .) .Values.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for PriorityClass. +*/}} +{{- define "PriorityClass.apiVersion" -}} +{{- if semverCompare ">=1.8-0, <1.11-0" .Capabilities.KubeVersion.Version -}} +{{- print "v1alpha1" -}} +{{- else if semverCompare ">=1.11-0, <1.14-0" .Capabilities.KubeVersion.Version -}} +{{- print "v1beta1" -}} +{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.Version -}} +{{- print "v1" -}} +{{- end -}} +{{- end -}} + +{{/* +Compose an fully qualified image. example: ghcr.io/teraslice:v0.87.0-nodev18.16.0 +*/}} +{{- define "teraslice.image" -}} +{{- if .Values.image.tag -}} +{{ .Values.image.repository }}:{{ .Values.image.tag }} +{{- else -}} +{{ .Values.image.repository }}:{{ .Chart.AppVersion }}-node{{ .Values.image.nodeVersion }} +{{- end -}} +{{- end -}} + + +{{/* +Create teraslice master base config +*/}} +{{- define "teraslice.masterConfig" -}} +{{- with .Values.terafoundation }} +terafoundation: + {{- toYaml . | nindent 2 }} +{{- end }} + +{{- with .Values.stats }} +stats: + {{- toYaml . | nindent 2 }} +{{- end }} + +teraslice: + state: + connection: {{ .Values.stateConnection }} + master: true + master_hostname: {{ template "teraslice.fullname" . }} + name: {{ .Release.Name }} + kubernetes_namespace: {{ .Release.Namespace }} + kubernetes_image: {{ include "teraslice.image" . }} + kubernetes_config_map_name: {{ template "teraslice.fullname" . }}-worker + {{- if and .Values.priorityClass.create (not .Values.master.teraslice.kubernetes_priority_class_name) }} + kubernetes_priority_class_name: {{ template "teraslice.fullname" . }} + {{- end }} + {{- if .Values.persistence.enabled }} + assets_volume: {{ template "teraslice.fullname" . }}-assets + {{- end }} +{{- end -}} + + +{{/* +Create teraslice master config user overrides +*/}} +{{- define "teraslice.masterConfigUser" -}} +teraslice: +{{- with .Values.master.teraslice }} + {{- toYaml . | nindent 2 }} +{{- end }} +{{- end -}} + + +{{/* +Create teraslice worker base config +*/}} +{{- define "teraslice.workerConfig" -}} +{{- with .Values.terafoundation }} +terafoundation: + {{- toYaml . | nindent 2 }} +{{- end }} + +{{- with .Values.stats }} +stats: + {{- toYaml . | nindent 2 }} +{{- end }} + +teraslice: + state: + connection: {{ .Values.stateConnection }} + master: false + master_hostname: {{ template "teraslice.fullname" . }} + name: {{ .Release.Name }} + kubernetes_namespace: {{ .Release.Namespace }} + kubernetes_image: {{ include "teraslice.image" . }} +{{- end -}} + +{{/* +Create teraslice worker config user overrides +*/}} +{{- define "teraslice.workerConfigUser" -}} +teraslice: +{{- with .Values.worker.teraslice }} + {{- toYaml . | nindent 2 }} +{{- end }} +{{- end -}} diff --git a/helm/teraslice/templates/configmap-master.yaml b/helm/teraslice/templates/configmap-master.yaml new file mode 100644 index 00000000000..6a2524fb212 --- /dev/null +++ b/helm/teraslice/templates/configmap-master.yaml @@ -0,0 +1,13 @@ +{{- $baseConfig := include "teraslice.masterConfig" . | fromYaml -}} +{{- $userConfig := include "teraslice.masterConfigUser" . | fromYaml -}} +{{- $mergedConfig := mustMergeOverwrite (dict) $baseConfig $userConfig -}} + +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "teraslice.fullname" . }}-master + labels: + {{- include "teraslice.master.labels" . | nindent 4 }} +data: + teraslice.yaml: | + {{- toYaml $mergedConfig | nindent 4 }} diff --git a/helm/teraslice/templates/configmap-worker.yaml b/helm/teraslice/templates/configmap-worker.yaml new file mode 100644 index 00000000000..a0f365edbb6 --- /dev/null +++ b/helm/teraslice/templates/configmap-worker.yaml @@ -0,0 +1,13 @@ +{{- $baseConfig := include "teraslice.workerConfig" . | fromYaml -}} +{{- $userConfig := include "teraslice.workerConfigUser" . | fromYaml -}} +{{- $mergedConfig := mustMergeOverwrite (dict) $baseConfig $userConfig -}} + +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "teraslice.fullname" . }}-worker + labels: + {{- include "teraslice.worker.labels" . | nindent 4 }} +data: + teraslice.yaml: | + {{- toYaml $mergedConfig | nindent 4 }} diff --git a/helm/teraslice/templates/deployment.yaml b/helm/teraslice/templates/deployment.yaml new file mode 100644 index 00000000000..2f665304b88 --- /dev/null +++ b/helm/teraslice/templates/deployment.yaml @@ -0,0 +1,144 @@ +{{- if and .Values.terafoundation.prom_metrics_enabled .Values.exporter.enabled }} +{{- fail "Do not enable both internal and external metrics (terafoundation.prom_metrics_enabled and exporter.enabled)" }} +{{- end}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "teraslice.fullname" . }}-master + labels: + {{- include "teraslice.master.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + strategy: + type: {{ .Values.strategyType }} + {{- if (eq "Recreate" .Values.strategyType) }} + rollingUpdate: null + {{- end }} + selector: + matchLabels: + {{- include "teraslice.master.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "teraslice.master.selectorLabels" . | nindent 8 }} + annotations: + checksum/config: {{ include (print $.Template.BasePath "/configmap-master.yaml") . | sha256sum }} + {{- with .Values.podAnnotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "teraslice.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + {{- if .Values.priorityClass.create }} + priorityClassName: {{ template "teraslice.fullname" . }} + {{- else if .Values.master.teraslice.kubernetes_priority_class_name }} + priorityClassName: {{ .Values.master.teraslice.kubernetes_priority_class_name }} + {{- end }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: {{ include "teraslice.image" . }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + env: + {{- range $key, $value := .Values.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + ports: + - name: api + containerPort: 5678 + protocol: TCP + - name: master + containerPort: 45678 + protocol: TCP + - name: assets + containerPort: 45679 + protocol: TCP + {{- if .Values.terafoundation.prom_metrics_enabled}} + - name: metrics + containerPort: {{ .Values.terafoundation.prom_metrics_port }} + protocol: TCP + {{- end }} + volumeMounts: + - name: config + mountPath: /app/config + - name: teraslice-assets + mountPath: /app/assets + {{- range .Values.dataVolumes }} + - name: {{ .name }} + mountPath: {{ .path }} + {{- end }} + {{- if .Values.extraVolumeMounts }} + {{- tpl .Values.extraVolumeMounts . | nindent 12 }} + {{- end }} + {{- with .Values.livenessProbe }} + livenessProbe: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.readinessProbe }} + readinessProbe: + {{- toYaml . | nindent 12 }} + {{- end }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- if .Values.extraContainers }} + {{- tpl .Values.extraContainers . | nindent 8 }} + {{- end }} + {{- if .Values.exporter.enabled }} + - name: {{ .Chart.Name }}-exporter + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: {{ .Values.exporter.image.repository }}:{{ .Values.exporter.image.tag }} + imagePullPolicy: {{ .Values.exporter.image.pullPolicy }} + env: + {{- range $key, $value := .Values.exporter.env }} + - name: "{{ $key }}" + value: "{{ $value }}" + {{- end }} + ports: + - name: metrics + containerPort: {{ .Values.exporter.env.PORT }} + protocol: TCP + resources: + {{- toYaml .Values.exporter.resources | nindent 12 }} + {{- end }} + volumes: + - name: config + configMap: + name: {{ template "teraslice.fullname" . }}-master + items: + - key: teraslice.yaml + path: teraslice.yaml + - name: teraslice-assets + {{- if .Values.persistence.enabled }} + persistentVolumeClaim: + claimName: {{ template "teraslice.fullname" . }}-assets + {{- else }} + emptyDir: {} + {{- end }} + {{- range .Values.dataVolumes }} + - name: {{ .name }} + persistentVolumeClaim: + claimName: {{ .name }} + {{- end }} + {{- if .Values.extraVolumes }} + {{- tpl .Values.extraVolumes . | nindent 8 }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/helm/teraslice/templates/ingress.yaml b/helm/teraslice/templates/ingress.yaml new file mode 100644 index 00000000000..b2cc87fdd02 --- /dev/null +++ b/helm/teraslice/templates/ingress.yaml @@ -0,0 +1,61 @@ +{{- if .Values.ingress.enabled -}} +{{- $fullName := include "teraslice.fullname" . -}} +{{- $svcPort := .Values.service.port -}} +{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.Version)) }} + {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} + {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} + {{- end }} +{{- end }} +{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.Version -}} +apiVersion: networking.k8s.io/v1 +{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.Version -}} +apiVersion: networking.k8s.io/v1beta1 +{{- else -}} +apiVersion: extensions/v1beta1 +{{- end }} +kind: Ingress +metadata: + name: {{ $fullName }} + labels: + {{- include "teraslice.master.labels" . | nindent 4 }} + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.Version) }} + ingressClassName: {{ .Values.ingress.className }} + {{- end }} + {{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.Version) }} + pathType: {{ .pathType }} + {{- end }} + backend: + {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.Version }} + service: + name: {{ $fullName }} + port: + number: {{ $svcPort }} + {{- else }} + serviceName: {{ $fullName }} + servicePort: {{ $svcPort }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} diff --git a/helm/teraslice/templates/persistence.yaml b/helm/teraslice/templates/persistence.yaml new file mode 100644 index 00000000000..7cb98a78b22 --- /dev/null +++ b/helm/teraslice/templates/persistence.yaml @@ -0,0 +1,25 @@ +{{- if .Values.persistence.enabled }} +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: {{ template "teraslice.fullname" . }}-assets + labels: + {{- include "teraslice.labels" . | nindent 4 }} +spec: + storageClassName: {{ .Values.persistence.class }} + {{- with .Values.persistence.accessModes }} + accessModes: + {{- toYaml . | nindent 4 }} + {{- end }} + resources: + requests: + storage: {{ .Values.persistence.size }} + #selector: {} + # This selector be needed if we wanted to re-use a Volume. The question remains + # on how to get the Volume automatically labeled. We're hoping to never need + # to do that (ie `helm delete`) unless we really want to delete the cluster & + # its backing volumes. + # + # selector: + # release: {{ .Release.Name }} +{{- end }} diff --git a/helm/teraslice/templates/podmonitor.yaml b/helm/teraslice/templates/podmonitor.yaml new file mode 100644 index 00000000000..cd8accbde9b --- /dev/null +++ b/helm/teraslice/templates/podmonitor.yaml @@ -0,0 +1,46 @@ +{{- if .Values.podMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: PodMonitor +metadata: + name: {{ include "teraslice.fullname" . }}-job-metrics + labels: + {{- include "teraslice.labels" . | nindent 4 }} + {{- with .Values.podMonitor.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.podMonitor.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + jobLabel: {{ .Values.podMonitor.jobLabel | quote }} + namespaceSelector: + matchNames: + - {{ .Release.Namespace }} + selector: + matchLabels: + {{- with .Values.podMonitor.matchLabels }} + {{- toYaml . | nindent 6 }} + {{- end }} + + {{- with .Values.podMonitor.podTargetLabels }} + podTargetLabels: + {{- toYaml . | nindent 4 }} + {{- end }} + podMetricsEndpoints: + - honorLabels: false + honorTimestamps: true + path: /metrics + port: metrics + {{- if .Values.podMonitor.interval }} + interval: {{ .Values.podMonitor.interval }} + {{- end }} + {{- if .Values.podMonitor.metricRelabelings }} + metricRelabelings: + {{- toYaml .Values.podMonitor.metricRelabelings | nindent 4 }} + {{- end }} + {{- if .Values.podMonitor.relabelings }} + relabelings: + {{- toYaml .Values.podMonitor.relabelings | nindent 4 }} + {{- end }} +{{- end }} diff --git a/helm/teraslice/templates/priorityclass.yaml b/helm/teraslice/templates/priorityclass.yaml new file mode 100644 index 00000000000..979efcead75 --- /dev/null +++ b/helm/teraslice/templates/priorityclass.yaml @@ -0,0 +1,16 @@ +{{- if and .Values.priorityClass.create .Values.master.teraslice.kubernetes_priority_class_name }} +{{ fail "Cannot specify master.teraslice.kubernetes_priority_class_name when priorityClass.create is true." }} +{{- end }} + +{{- if and .Values.priorityClass.create (not .Values.master.teraslice.kubernetes_priority_class_name) }} +apiVersion: scheduling.k8s.io/{{ template "PriorityClass.apiVersion" . }} +kind: PriorityClass +metadata: + name: {{ template "teraslice.fullname" . }} + labels: + {{- include "teraslice.master.labels" . | nindent 4 }} +value: {{ .Values.priorityClass.value }} +preemptionPolicy: {{ .Values.priorityClass.preemptionPolicy }} +globalDefault: false +description: "Priority class that the Teraslice master, execution controller, and stateful workers should run with" +{{- end }} diff --git a/helm/teraslice/templates/prometheusrules.yaml b/helm/teraslice/templates/prometheusrules.yaml new file mode 100644 index 00000000000..2a02eb226b6 --- /dev/null +++ b/helm/teraslice/templates/prometheusrules.yaml @@ -0,0 +1,13 @@ +{{- if .Values.prometheusRule.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ include "teraslice.fullname" . }} + labels: + {{- include "teraslice.master.labels" . | nindent 4 }} +spec: + groups: + - name: {{ include "teraslice.fullname" . }} + rules: + {{- toYaml .Values.prometheusRules.rules | nindent 8 }} +{{- end }} diff --git a/helm/teraslice/templates/role.yaml b/helm/teraslice/templates/role.yaml new file mode 100644 index 00000000000..bfe7a773fc7 --- /dev/null +++ b/helm/teraslice/templates/role.yaml @@ -0,0 +1,11 @@ +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: teraslice-all-{{ .Release.Name }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "teraslice.master.labels" . | nindent 4 }} +rules: + - apiGroups: ["*"] + resources: ["*"] + verbs: ["*"] diff --git a/helm/teraslice/templates/rolebinding.yaml b/helm/teraslice/templates/rolebinding.yaml new file mode 100644 index 00000000000..84635e59446 --- /dev/null +++ b/helm/teraslice/templates/rolebinding.yaml @@ -0,0 +1,15 @@ +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: teraslice-all-{{ .Release.Name }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "teraslice.master.labels" . | nindent 4 }} +subjects: + - kind: ServiceAccount + name: {{ include "teraslice.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: Role + name: teraslice-all-{{ .Release.Name }} + apiGroup: "rbac.authorization.k8s.io" diff --git a/helm/teraslice/templates/service.yaml b/helm/teraslice/templates/service.yaml new file mode 100644 index 00000000000..818bcbcdd4e --- /dev/null +++ b/helm/teraslice/templates/service.yaml @@ -0,0 +1,57 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "teraslice.fullname" . }} + labels: + {{- include "teraslice.master.labels" . | nindent 4 }} + {{- with .Values.service.labels }} + {{ toYaml . | indent 4 }} + {{- end }} + {{- with .Values.service.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if (or (eq .Values.service.type "ClusterIP") (empty .Values.service.type)) }} + type: ClusterIP + {{- if .Values.service.clusterIP }} + clusterIP: {{ .Values.service.clusterIP }} + {{- end}} + {{- else if eq .Values.service.type "LoadBalancer" }} + type: {{ .Values.service.type }} + {{- if .Values.service.loadBalancerIP }} + loadBalancerIP: {{ .Values.service.loadBalancerIP }} + {{- end }} + {{- with .Values.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: + {{- toYaml . | nindent 4 }} + {{- end -}} + {{- else }} + type: {{ .Values.service.type }} + {{- end }} + {{- with .Values.service.externalIPs }} + externalIPs: + {{- toYaml . | nindent 4 }} + {{- end }} + ports: + - port: {{ .Values.service.port }} + targetPort: api + protocol: TCP + name: api + {{- if .Values.service.nodePort }} + nodePort: {{ .Values.service.nodePort }} + {{- end }} + {{- if .Values.terafoundation.prom_metrics_enabled }} + - port: {{ .Values.terafoundation.prom_metrics_port }} + targetPort: metrics + protocol: TCP + name: metrics + {{- end }} + {{- if .Values.exporter.enabled }} + - port: {{ .Values.exporter.env.PORT }} + targetPort: metrics + protocol: TCP + name: metrics + {{- end }} + selector: + {{- include "teraslice.master.selectorLabels" . | nindent 4 }} diff --git a/helm/teraslice/templates/serviceaccount.yaml b/helm/teraslice/templates/serviceaccount.yaml new file mode 100644 index 00000000000..336dd6ef834 --- /dev/null +++ b/helm/teraslice/templates/serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "teraslice.serviceAccountName" . }} + labels: + {{- include "teraslice.master.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end -}} diff --git a/helm/teraslice/templates/servicemonitor.yaml b/helm/teraslice/templates/servicemonitor.yaml new file mode 100644 index 00000000000..be3c9710a72 --- /dev/null +++ b/helm/teraslice/templates/servicemonitor.yaml @@ -0,0 +1,30 @@ +{{- if .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ include "teraslice.fullname" . }} + labels: + {{- include "teraslice.master.labels" . | nindent 4 }} +spec: + endpoints: + - path: /cluster/stats + port: api + interval: {{ .Values.serviceMonitor.interval }} + metricRelabelings: + {{- toYaml .Values.serviceMonitor.metricRelabelings | nindent 8 }} + relabelings: + {{- toYaml .Values.serviceMonitor.relabelings | nindent 8 }} + {{- if or .Values.terafoundation.prom_metrics_enabled .Values.exporter.enabled }} + - path: /metrics + port: metrics + interval: {{ .Values.serviceMonitor.interval }} + metricRelabelings: + {{- toYaml .Values.serviceMonitor.metricRelabelings | nindent 8 }} + relabelings: + {{- toYaml .Values.serviceMonitor.relabelings | nindent 8 }} + {{- end }} + jobLabel: app.kubernetes.io/instance + selector: + matchLabels: + {{- include "teraslice.master.selectorLabels" . | nindent 6 }} +{{- end }} diff --git a/helm/teraslice/templates/tests/test-connection.yaml b/helm/teraslice/templates/tests/test-connection.yaml new file mode 100644 index 00000000000..d4bee831b66 --- /dev/null +++ b/helm/teraslice/templates/tests/test-connection.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "teraslice.fullname" . }}-test-connection" + labels: + {{- include "teraslice.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test-success +spec: + containers: + - name: wget + image: {{ .Values.busybox.repository }}:{{ .Values.busybox.tag }} + imagePullPolicy: {{ .Values.busybox.pullPolicy }} + command: ['wget'] + args: ['{{ include "teraslice.fullname" . }}:{{ .Values.service.port }}'] + restartPolicy: Never diff --git a/helm/teraslice/values.yaml b/helm/teraslice/values.yaml new file mode 100644 index 00000000000..cb83b094448 --- /dev/null +++ b/helm/teraslice/values.yaml @@ -0,0 +1,239 @@ +# Default values for teraslice. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 + +image: + repository: ghcr.io/terascope/teraslice + # image tag defaults to chart appVersion + # tag: v2.6.2-nodev22.9.0 + pullPolicy: IfNotPresent + # node version will be inserted into the image tag (which defaults the chart version) + # setting the image tag will ignore this value + nodeVersion: v22.9.0 + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +extraContainers: [] + +# Terasclice master env +env: {} + +# Teraslice State Connection / es util service +stateConnection: default + +# Create Pod Priority Class for this cluster +# assigns kubernetes_priority_class_name +# https://terascope.github.io/teraslice/docs/configuration/clustering-k8s#kubernetes-specific-configuration-settings +# if you want to use an existing priortyClass, set create to false and assign kubernetes_priority_class_name yourself +priorityClass: + create: false + preemptionPolicy: Never + value: 9999 + +# Teraslice Master Config Options +master: + teraslice: + assets_directory: /app/assets + workers: 0 + +# Teraslice Worker Config Options +worker: + teraslice: + assets_directory: /app/assets + +# Terafoundation +# Everything under terafoundation key will used as the master and worker config +terafoundation: + environment: production + log_level: info + # internal metrics exporter. Do not combine with the external metrics server + # exporter.enabled: false + prom_metrics_enabled: false + prom_metrics_port: 3333 + prom_metrics_add_default: true + # log_path: /app/logs + # logging: + # - file + connectors: {} + # elasticsearch: + # default: + # host: + # - example.local:9200 + # suggestCompression: true + # sniffOnStart: true + # sniffOnConnectionFault: false + # # 1m in milliseconds + # sniffInterval: 60000 + # elasticsearch-next: + # default: + # node: + # - "http://example.ns:9200" + # kafka: + # kafka_gen1: + # brokers: > + # example1.local:9092, + # example2.local:9092, + # example3.local:9092 + stats: {} + # elasticsearch: + # connection: default + # enabled: true + +serviceAccount: + # Specifies whether a service account should be created + create: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: + +podSecurityContext: {} + # fsGroup: 2000 + +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +strategyType: Recreate + +# livenessProbe: +# httpGet: +# path: /cluster/stats +# port: 5678 +# initialDelaySeconds: 120 + +# readinessProbe: +# httpGet: +# path: /cluster/stats +# port: 5678 +# initialDelaySeconds: 60 + +service: + type: ClusterIP + port: 5678 + annotations: {} + labels: {} + nodePort: null + # loadBalancerIP + # loadBalancerSourceRanges + # externalIPs + +ingress: + enabled: false + className: "" + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + hosts: + - host: teraslice.local + paths: + - path: / + pathType: ImplementationSpecific + tls: [] + # - secretName: teraslice-tls + # hosts: + # - teraslice.local + +resources: {} + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +nodeSelector: {} + +tolerations: [] + +affinity: {} + +# Type & size of persistent storage volume. +persistence: + enabled: false + #class: glusterfs + size: 20Gi + accessModes: + - ReadWriteMany + +extraVolumes: [] + # - name: extras + # emptyDir: {} + +extraVolumeMounts: [] + # - name: extras + # mountPath: /usr/share/extras + # readOnly: true + +# external exporter. Do not combine with the internal metrics server +# terafoundation.prom_metrics_enabled: false +exporter: + enabled: false + image: + repository: terascope/teraslice-exporter + tag: v0.4.0 + pullPolicy: IfNotPresent + env: + TERASLICE_URL: http://localhost:5678 + PORT: 8080 + +serviceMonitor: + enabled: false + interval: 60s + metricRelabelings: [] + # - action: replace + # regex: teraslice-(.*) + # replacement: $1 + # sourceLabels: [cluster] + # targetLabel: ts_cluster + # - action: labeldrop + # regex: (endpoint|cluster) + relabelings: [] + # - sourceLabels: [__meta_kubernetes_pod_node_name] + # targetLabel: instance + rules: null + +# Pod Monitor used by teraslice job metric api +podMonitor: + enabled: false + labels: {} + #scrape.prometheus.io/instance: prometheus-instance + annotations: {} + jobLabel: app.kubernetes.io/instance + interval: 60s + podTargetLabels: + - app.kubernetes.io/instance + - app.kubernetes.io/name + - app.kubernetes.io/component + - teraslice.terascope.io/jobId + - teraslice.terascope.io/jobName + matchLabels: {} + #job.teraslice.terascope.io/scrape-target: "true" + #job.teraslice.terascope.io/scrape-prometheus-instance: "prometheus-instance" + metricRelabelings: [] + relabelings: [] + +# Prometheus Operator alertmanager alerts +prometheusRule: + enabled: false + rules: [] + # - alert: Example + # expr: metric == 1 + # for: 1m + # labels: + # severity: warning + +# Test images +busybox: + repository: busybox + tag: latest + pullPolicy: IfNotPresent