diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 71a3bb6735f..eeda236200e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -74,15 +74,14 @@ jobs: - name: Test run: yarn test --suite unit -- - teraslice-elasticsearch-tests: + e2e-k8s-tests: runs-on: ubuntu-latest needs: [compute-node-version-vars, verify-build, cache-docker-images] strategy: # opensearch is finiky, keep testing others if it fails fail-fast: false matrix: - node-version: ${{ fromJSON(needs.compute-node-version-vars.outputs.NODE_VERSIONS) }} - search-version: [elasticsearch6, elasticsearch7, opensearch1, opensearch2] + node-version: ${{ fromJSON(needs.compute-node-version-vars.outputs.NODE_VERSIONS_EXT_K8S) }} steps: - name: Check out code uses: actions/checkout@v4 @@ -115,19 +114,27 @@ jobs: path: /tmp/docker_cache key: docker-images-${{ hashFiles('./images/image-list.txt') }} - - name: Test ${{ matrix.search-version }} - run: yarn test:${{ matrix.search-version }} - working-directory: ./packages/teraslice + - name: Compile e2e code + run: yarn build + working-directory: ./e2e - elasticsearch-store-tests: + - name: Install Kind and Kubectl + uses: helm/kind-action@v1.10.0 + with: + install_only: "true" + + - name: Test k8s elasticsearch7 + run: NODE_VERSION=${{ matrix.node-version }} yarn test:k8s + working-directory: ./e2e + + e2e-k8s-v2-tests: runs-on: ubuntu-latest needs: [compute-node-version-vars, verify-build, cache-docker-images] strategy: # opensearch is finiky, keep testing others if it fails fail-fast: false matrix: - node-version: ${{ fromJSON(needs.compute-node-version-vars.outputs.NODE_VERSIONS) }} - search-version: [elasticsearch6, elasticsearch7, opensearch1, opensearch2] + node-version: ${{ fromJSON(needs.compute-node-version-vars.outputs.NODE_VERSIONS_EXT_K8S) }} steps: - name: Check out code uses: actions/checkout@v4 @@ -160,43 +167,46 @@ jobs: path: /tmp/docker_cache key: docker-images-${{ hashFiles('./images/image-list.txt') }} - - name: Test ${{ matrix.search-version }} - run: yarn test:${{ matrix.search-version }} - working-directory: ./packages/elasticsearch-store - - lint-and-sync: - runs-on: ubuntu-latest - needs: [compute-node-version-vars, verify-build, cache-docker-images] - # will remove the checkout, build and setup when the artifact is made to just - # test the linting and syncing of the codebase - steps: - - name: Check out code - uses: actions/checkout@v4 - - - name: Setup Node - uses: actions/setup-node@v4 - with: - node-version: ${{ fromJSON(needs.compute-node-version-vars.outputs.NODE_VERSION_MAIN) }} - cache: 'yarn' + - name: Compile e2e code + run: yarn build + working-directory: ./e2e - - name: Install and build packages - run: yarn && yarn setup + - name: Install Kind and Kubectl + uses: helm/kind-action@v1.10.0 + with: + install_only: "true" - - name: Lint codebase - run: yarn lint + - name: Install Helmfile + run: | + VERSION=$(curl -s https://api.github.com/repos/helmfile/helmfile/releases/latest | jq -r '.tag_name') + VERSION_NO_V=${VERSION#v} + curl -L "https://github.com/helmfile/helmfile/releases/download/${VERSION}/helmfile_${VERSION_NO_V}_linux_amd64.tar.gz" -o helmfile.tgz + tar -xzf helmfile.tgz + chmod +x helmfile + mv helmfile /usr/local/bin/helmfile + helm plugin install https://github.com/databus23/helm-diff + helmfile version + helm version + + - name: Test CLI Commands + run: | + command -v kind + command -v kubectl + command -v helm + command -v helmfile - - name: Sync codebase - run: yarn sync --verify + - name: Test k8s V2 opensearch2 + run: NODE_VERSION=${{ matrix.node-version }} yarn test:k8sV2Helmfile + working-directory: ./e2e - elasticsearch-api-tests: + e2e-external-storage-tests: runs-on: ubuntu-latest needs: [compute-node-version-vars, verify-build, cache-docker-images] strategy: # opensearch is finiky, keep testing others if it fails fail-fast: false matrix: - node-version: ${{ fromJSON(needs.compute-node-version-vars.outputs.NODE_VERSIONS) }} - search-version: [elasticsearch6, elasticsearch7, opensearch1, opensearch2] + node-version: ${{ fromJSON(needs.compute-node-version-vars.outputs.NODE_VERSIONS_EXT_STORAGE) }} steps: - name: Check out code uses: actions/checkout@v4 @@ -229,19 +239,22 @@ jobs: path: /tmp/docker_cache key: docker-images-${{ hashFiles('./images/image-list.txt') }} - - name: Test ${{ matrix.search-version }} - run: yarn test:${{ matrix.search-version }} - working-directory: ./packages/elasticsearch-api + - name: Compile e2e code + run: yarn build + working-directory: ./e2e - e2e-tests: + - name: Test external Asset Storage opensearch1 + run: NODE_VERSION=${{ matrix.node-version }} yarn test:s3AssetStorage + working-directory: ./e2e + + e2e-external-storage-tests-encrypted: runs-on: ubuntu-latest needs: [compute-node-version-vars, verify-build, cache-docker-images] strategy: # opensearch is finiky, keep testing others if it fails fail-fast: false matrix: - node-version: ${{ fromJSON(needs.compute-node-version-vars.outputs.NODE_VERSIONS) }} - search-version: [elasticsearch6, elasticsearch7, opensearch1, opensearch2] + node-version: ${{ fromJSON(needs.compute-node-version-vars.outputs.NODE_VERSIONS_EXT_STORAGE) }} steps: - name: Check out code uses: actions/checkout@v4 @@ -274,22 +287,35 @@ jobs: path: /tmp/docker_cache key: docker-images-${{ hashFiles('./images/image-list.txt') }} + - name: Install mkcert + run: curl -JLO "https://dl.filippo.io/mkcert/latest?for=linux/amd64" && sudo chmod 777 mkcert-v*-linux-amd64 && sudo cp mkcert-v*-linux-amd64 /usr/local/bin/mkcert + + - name: Install grep + run: sudo apt update && sudo apt install grep + + - name: Check mkcert + run: command -v mkcert + + - name: Check grep + run: command -v grep + - name: Compile e2e code run: yarn build working-directory: ./e2e - - name: Test ${{ matrix.search-version }} - run: NODE_VERSION=${{ matrix.node-version }} yarn test:${{ matrix.search-version }} + - name: Test external Asset Storage opensearch1 + run: ENCRYPT_MINIO=true NODE_VERSION=${{ matrix.node-version }} yarn test:s3AssetStorage working-directory: ./e2e - e2e-k8s-tests: + teraslice-elasticsearch-tests: runs-on: ubuntu-latest needs: [compute-node-version-vars, verify-build, cache-docker-images] strategy: # opensearch is finiky, keep testing others if it fails fail-fast: false matrix: - node-version: ${{ fromJSON(needs.compute-node-version-vars.outputs.NODE_VERSIONS_EXT_K8S) }} + node-version: ${{ fromJSON(needs.compute-node-version-vars.outputs.NODE_VERSIONS) }} + search-version: [elasticsearch6, elasticsearch7, opensearch1, opensearch2] steps: - name: Check out code uses: actions/checkout@v4 @@ -322,27 +348,19 @@ jobs: path: /tmp/docker_cache key: docker-images-${{ hashFiles('./images/image-list.txt') }} - - name: Compile e2e code - run: yarn build - working-directory: ./e2e - - - name: Install Kind and Kubectl - uses: helm/kind-action@v1.10.0 - with: - install_only: "true" - - - name: Test k8s elasticsearch7 - run: NODE_VERSION=${{ matrix.node-version }} yarn test:k8s - working-directory: ./e2e + - name: Test ${{ matrix.search-version }} + run: yarn test:${{ matrix.search-version }} + working-directory: ./packages/teraslice - e2e-k8s-v2-tests: + elasticsearch-store-tests: runs-on: ubuntu-latest needs: [compute-node-version-vars, verify-build, cache-docker-images] strategy: # opensearch is finiky, keep testing others if it fails fail-fast: false matrix: - node-version: ${{ fromJSON(needs.compute-node-version-vars.outputs.NODE_VERSIONS_EXT_K8S) }} + node-version: ${{ fromJSON(needs.compute-node-version-vars.outputs.NODE_VERSIONS) }} + search-version: [elasticsearch6, elasticsearch7, opensearch1, opensearch2] steps: - name: Check out code uses: actions/checkout@v4 @@ -375,27 +393,43 @@ jobs: path: /tmp/docker_cache key: docker-images-${{ hashFiles('./images/image-list.txt') }} - - name: Compile e2e code - run: yarn build - working-directory: ./e2e + - name: Test ${{ matrix.search-version }} + run: yarn test:${{ matrix.search-version }} + working-directory: ./packages/elasticsearch-store - - name: Install Kind and Kubectl - uses: helm/kind-action@v1.10.0 - with: - install_only: "true" + lint-and-sync: + runs-on: ubuntu-latest + needs: [compute-node-version-vars, verify-build, cache-docker-images] + # will remove the checkout, build and setup when the artifact is made to just + # test the linting and syncing of the codebase + steps: + - name: Check out code + uses: actions/checkout@v4 - - name: Test k8s V2 elasticsearch7 - run: NODE_VERSION=${{ matrix.node-version }} yarn test:k8sV2 - working-directory: ./e2e + - name: Setup Node + uses: actions/setup-node@v4 + with: + node-version: ${{ fromJSON(needs.compute-node-version-vars.outputs.NODE_VERSION_MAIN) }} + cache: 'yarn' - e2e-external-storage-tests: + - name: Install and build packages + run: yarn && yarn setup + + - name: Lint codebase + run: yarn lint + + - name: Sync codebase + run: yarn sync --verify + + elasticsearch-api-tests: runs-on: ubuntu-latest needs: [compute-node-version-vars, verify-build, cache-docker-images] strategy: # opensearch is finiky, keep testing others if it fails fail-fast: false matrix: - node-version: ${{ fromJSON(needs.compute-node-version-vars.outputs.NODE_VERSIONS_EXT_STORAGE) }} + node-version: ${{ fromJSON(needs.compute-node-version-vars.outputs.NODE_VERSIONS) }} + search-version: [elasticsearch6, elasticsearch7, opensearch1, opensearch2] steps: - name: Check out code uses: actions/checkout@v4 @@ -428,22 +462,19 @@ jobs: path: /tmp/docker_cache key: docker-images-${{ hashFiles('./images/image-list.txt') }} - - name: Compile e2e code - run: yarn build - working-directory: ./e2e - - - name: Test external Asset Storage opensearch1 - run: NODE_VERSION=${{ matrix.node-version }} yarn test:s3AssetStorage - working-directory: ./e2e + - name: Test ${{ matrix.search-version }} + run: yarn test:${{ matrix.search-version }} + working-directory: ./packages/elasticsearch-api - e2e-external-storage-tests-encrypted: + e2e-tests: runs-on: ubuntu-latest needs: [compute-node-version-vars, verify-build, cache-docker-images] strategy: # opensearch is finiky, keep testing others if it fails fail-fast: false matrix: - node-version: ${{ fromJSON(needs.compute-node-version-vars.outputs.NODE_VERSIONS_EXT_STORAGE) }} + node-version: ${{ fromJSON(needs.compute-node-version-vars.outputs.NODE_VERSIONS) }} + search-version: [elasticsearch6, elasticsearch7, opensearch1, opensearch2] steps: - name: Check out code uses: actions/checkout@v4 @@ -476,24 +507,12 @@ jobs: path: /tmp/docker_cache key: docker-images-${{ hashFiles('./images/image-list.txt') }} - - name: Install mkcert - run: curl -JLO "https://dl.filippo.io/mkcert/latest?for=linux/amd64" && sudo chmod 777 mkcert-v*-linux-amd64 && sudo cp mkcert-v*-linux-amd64 /usr/local/bin/mkcert - - - name: Install grep - run: sudo apt update && sudo apt install grep - - - name: Check mkcert - run: command -v mkcert - - - name: Check grep - run: command -v grep - - name: Compile e2e code run: yarn build working-directory: ./e2e - - name: Test external Asset Storage opensearch1 - run: ENCRYPT_MINIO=true NODE_VERSION=${{ matrix.node-version }} yarn test:s3AssetStorage + - name: Test ${{ matrix.search-version }} + run: NODE_VERSION=${{ matrix.node-version }} yarn test:${{ matrix.search-version }} working-directory: ./e2e check-docker-limit-after: diff --git a/e2e/helm/elasticsearch/README.md b/e2e/helm/elasticsearch/README.md new file mode 100644 index 00000000000..fb382a2e430 --- /dev/null +++ b/e2e/helm/elasticsearch/README.md @@ -0,0 +1,7 @@ +The es binaries in this directory are patched charts from https://artifacthub.io/packages/helm/elastic/elasticsearch (specifically charts versions 7.9.3 and 6.8.9) + +Since these charts are no longer maintained, they have incompatible kubernetes api resources that needed to be updated. + +The es charts for 6.8.9 and 7.9.3 have a lot files in them and I didn't want to push up 140+ files to the teraslice repo. + +In the future we can potentially place these charts in the es assets repo and submit them to artifact hub to get rid of these binaries. diff --git a/e2e/helm/elasticsearch/elasticsearch-6.8.9.tgz b/e2e/helm/elasticsearch/elasticsearch-6.8.9.tgz new file mode 100644 index 00000000000..cb28ba5f17b Binary files /dev/null and b/e2e/helm/elasticsearch/elasticsearch-6.8.9.tgz differ diff --git a/e2e/helm/elasticsearch/elasticsearch-7.9.3.tgz b/e2e/helm/elasticsearch/elasticsearch-7.9.3.tgz new file mode 100644 index 00000000000..2992f0e5f54 Binary files /dev/null and b/e2e/helm/elasticsearch/elasticsearch-7.9.3.tgz differ diff --git a/e2e/helm/helmfile.yaml b/e2e/helm/helmfile.yaml new file mode 100644 index 00000000000..e991a11322c --- /dev/null +++ b/e2e/helm/helmfile.yaml @@ -0,0 +1,70 @@ +environments: + default: + values: + - values.yaml +--- + +repositories: + - name: opensearch + url: https://opensearch-project.github.io/helm-charts/ + - name: minio + url: https://charts.min.io/ + +helmDefaults: + wait: true + +releases: + - name: opensearch1 + namespace: services-dev1 + chart: opensearch/opensearch + version: 2.17.1 + installed: {{ .Values | get "opensearch1.enabled" false }} + values: + - ./templates/os1.yaml.gotmpl + + - name: opensearch2 + namespace: services-dev1 + chart: opensearch/opensearch + version: 2.17.1 + installed: {{ .Values | get "opensearch2.enabled" true }} + values: + - ./templates/os2.yaml.gotmpl + + - name: elasticsearch6 + namespace: services-dev1 + chart: ./elasticsearch/elasticsearch-6.8.9.tgz + installed: {{ .Values | get "elasticsearch6.enabled" false }} + values: + - ./templates/es6.yaml.gotmpl + + - name: elasticsearch7 + namespace: services-dev1 + chart: ./elasticsearch/elasticsearch-7.9.3.tgz + installed: {{ .Values | get "elasticsearch7.enabled" false }} + values: + - ./templates/es7.yaml.gotmpl + + - name: minio + namespace: services-dev1 + chart: minio/minio + version: 5.3.0 + installed: {{ .Values | get "minio.enabled" false }} + values: + - ./templates/minio.yaml.gotmpl + + - name: kafka + namespace: services-dev1 + chart: ./kafka/cp-helm-charts-0.6.1.tgz + installed: {{ .Values | get "kafka.enabled" true }} + values: + - ./templates/kafka.yaml.gotmpl + + - name: teraslice + namespace: ts-dev1 + chart: ../../helm/teraslice + needs: + - services-dev1/{{ .Values | get "teraslice.stateCluster" "opensearch2" }} + values: + - ./templates/teraslice.yaml.gotmpl + labels: + app: teraslice diff --git a/e2e/helm/kafka/README.md b/e2e/helm/kafka/README.md new file mode 100644 index 00000000000..92710e5b2bc --- /dev/null +++ b/e2e/helm/kafka/README.md @@ -0,0 +1,3 @@ +The compressed archive in this directory is a modified copy of the confluentinc/cp-helm-charts from + +Since these charts are no longer maintained, they have incompatible kubernetes api resources that needed to be updated. diff --git a/e2e/helm/kafka/cp-helm-charts-0.6.1.tgz b/e2e/helm/kafka/cp-helm-charts-0.6.1.tgz new file mode 100644 index 00000000000..28cb64cea1e Binary files /dev/null and b/e2e/helm/kafka/cp-helm-charts-0.6.1.tgz differ diff --git a/e2e/helm/templates/es6.yaml.gotmpl b/e2e/helm/templates/es6.yaml.gotmpl new file mode 100644 index 00000000000..897888df768 --- /dev/null +++ b/e2e/helm/templates/es6.yaml.gotmpl @@ -0,0 +1,31 @@ +imageTag: {{ .Values | get "elasticsearch6.version" "6.8.6" }} + +image: {{ .Values | get "elasticsearch6.image" "docker.elastic.co/elasticsearch/elasticsearch" }} + +esMajorVersion: "6" + +replicas: {{ .Values | get "elasticsearch6.instances" 1 }} + +minimumMasterNodes: 1 + +clusterName: "elasticsearch6" + +esJavaOpts: {{ .Values | get "elasticsearch6.esJavaOpts" "-Xms1g -Xmx1g" }} + +resources: + requests: + cpu: "1000m" + memory: {{ .Values | get "elasticsearch6.memoryLimit" "2Gi" }} + limits: + cpu: "1000m" + memory: {{ .Values | get "elasticsearch6.memoryLimit" "2Gi" }} + +volumeClaimTemplate: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: {{ .Values | get "elasticsearch6.persistentVolumeSize" "30Gi" }} + +service: + type: NodePort + nodePort: "30200" diff --git a/e2e/helm/templates/es7.yaml.gotmpl b/e2e/helm/templates/es7.yaml.gotmpl new file mode 100644 index 00000000000..36e3b75805c --- /dev/null +++ b/e2e/helm/templates/es7.yaml.gotmpl @@ -0,0 +1,29 @@ +imageTag: {{ .Values | get "elasticsearch7.version" "7.9.3" }} + +esMajorVersion: "7" + +replicas: {{ .Values | get "elasticsearch7.instances" 1 }} + +minimumMasterNodes: 1 + +clusterName: "elasticsearch7" + +esJavaOpts: {{ .Values | get "elasticsearch7.esJavaOpts" "-Xms1g -Xmx1g" }} + +resources: + requests: + cpu: "1000m" + memory: {{ .Values | get "elasticsearch7.memoryLimit" "2Gi" }} + limits: + cpu: "1000m" + memory: {{ .Values | get "elasticsearch7.memoryLimit" "2Gi" }} + +volumeClaimTemplate: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: {{ .Values | get "elasticsearch7.persistentVolumeSize" "30Gi" }} + +service: + type: NodePort + nodePort: {{ .Values | get "elasticsearch7.nodePort" "30200" }} diff --git a/e2e/helm/templates/kafka.yaml.gotmpl b/e2e/helm/templates/kafka.yaml.gotmpl new file mode 100644 index 00000000000..54ad6c8324e --- /dev/null +++ b/e2e/helm/templates/kafka.yaml.gotmpl @@ -0,0 +1,30 @@ +cp-zookeeper: + imageTag: {{ .Values | get "zookeeper.version" "7.7.2" }} + servers: {{ .Values | get "zookeeper.instances" "1" }} + prometheus: + jmx: + enabled: false + +cp-kafka: + imageTag: {{ .Values | get "kafka.version" "7.7.2" }} + brokers: {{ .Values | get "kafka.brokers" "1" }} + configurationOverrides: + offsets.topic.replication.factor: {{ .Values | get "kafka.offsets.topic.replication.factor" "1" }} + prometheus: + jmx: + enabled: false + +cp-schema-registry: + enabled: false + +cp-kafka-rest: + enabled: false + +cp-kafka-connect: + enabled: false + +cp-ksql-server: + enabled: false + +cp-control-center: + enabled: false diff --git a/e2e/helm/templates/minio.yaml.gotmpl b/e2e/helm/templates/minio.yaml.gotmpl new file mode 100644 index 00000000000..d3570228105 --- /dev/null +++ b/e2e/helm/templates/minio.yaml.gotmpl @@ -0,0 +1,41 @@ +image: + repository: minio/minio + tag: {{ .Values | get "minio.version" "RELEASE.2024-08-29T01-40-52Z" }} + pullPolicy: IfNotPresent + +# tls: +# enabled: false +# ## Create a secret with private.key and public.crt files and pass that here. Ref: https://github.com/minio/minio/tree/master/docs/tls/kubernetes#2-create-kubernetes-secret +# certSecret: "" +# publicCrt: public.crt +# privateKey: private.key + +replicas: {{ .Values | get "minio.instances" 1 }} + +{{- if eq (.Values | get "minio.instances" 1) 1 }} +mode: standalone +{{- else }} +mode: distributed +{{- end }} + +minioAPIPort: "9000" +minioConsolePort: "9001" +rootUser: {{ .Values | get "minio.rootUser" "minioadmin" }} +rootPassword: {{ .Values | get "minio.rootPassword" "minioadmin" }} + +consoleService: + type: NodePort + port: 9001 + nodePort: 30901 + +service: + type: NodePort + port: 9000 + nodePort: 30900 + +resources: + requests: + memory: 256Mi + +persistence: + size: {{ .Values | get "minio.persistentVolumeSize" "50Gi" }} diff --git a/e2e/helm/templates/os1.yaml.gotmpl b/e2e/helm/templates/os1.yaml.gotmpl new file mode 100644 index 00000000000..91132be24e3 --- /dev/null +++ b/e2e/helm/templates/os1.yaml.gotmpl @@ -0,0 +1,36 @@ +replicas: {{ .Values | get "opensearch1.instances" 1 }} + +{{- if eq (.Values | get "opensearch1.instances" 1) 1 }} +singleNode: true +{{- else }} +singleNode: false +{{- end }} + +image: + tag: {{ .Values | get "opensearch1.version" "1.3.11" }} + +service: + type: NodePort + port: 9200 + nodePort: 30210 + +config: + opensearch.yml: + plugins: + security: + disabled: true + {{- if eq (.Values | get "opensearch1.instances" 1) 1 }} + discovery.type: single-node + {{- end }} + +clusterName: opensearch1-cluster + +masterService: "opensearch1" + +resources: + requests: + cpu: "1000m" + memory: {{ .Values | get "opensearch1.memoryLimit" "100Mi" }} + +persistence: + size: {{ .Values | get "opensearch1.persistentVolumeSize" "8Gi" }} diff --git a/e2e/helm/templates/os2.yaml.gotmpl b/e2e/helm/templates/os2.yaml.gotmpl new file mode 100644 index 00000000000..a7762b90719 --- /dev/null +++ b/e2e/helm/templates/os2.yaml.gotmpl @@ -0,0 +1,39 @@ +replicas: {{ .Values | get "opensearch2.instances" 1 }} + +{{- if eq (.Values | get "opensearch2.instances" 1) 1 }} +singleNode: true +{{- else }} +singleNode: false +{{- end }} + +image: + tag: {{ .Values | get "opensearch2.version" "2.15.0" }} + +service: + type: NodePort + port: 9200 + nodePort: 30210 + +config: + opensearch.yml: + {{- if eq (.Values | get "opensearch2.instances" 1) 1 }} + discovery.type: single-node + {{- end }} + +extraEnvs: + - name: DISABLE_SECURITY_PLUGIN + value: "true" + +clusterName: opensearch2-cluster + +masterService: "opensearch2" + +opensearchJavaOpts: {{ .Values | get "opensearch2.esJavaOpts" "-Xmx512M -Xms512M" }} + +resources: + requests: + cpu: "1000m" + memory: {{ .Values | get "opensearch2.memoryLimit" "100Mi" }} + +persistence: + size: {{ .Values | get "opensearch2.persistentVolumeSize" "8Gi" }} diff --git a/e2e/helm/templates/teraslice.yaml.gotmpl b/e2e/helm/templates/teraslice.yaml.gotmpl new file mode 100644 index 00000000000..db64c013039 --- /dev/null +++ b/e2e/helm/templates/teraslice.yaml.gotmpl @@ -0,0 +1,79 @@ +replicaCount: 1 + +image: + repository: {{ .Values | get "teraslice.image.repository" }} + tag: {{ .Values | get "teraslice.image.tag" }} + +service: + type: NodePort + nodePort: 30678 + targetPort: 5678 + +terafoundation: + environment: development + log_level: info + workers: 1 + connectors: + elasticsearch-next: + # Set the first OS/ES that is enabled as our default + default: + node: + {{- if eq (.Values | get "teraslice.stateCluster" "none") "opensearch2" }} + - "http://opensearch2.services-dev1:9200" + {{- else if eq (.Values | get "teraslice.stateCluster" "none") "opensearch1" }} + - "http://opensearch1.services-dev1:9200" + {{- else if eq (.Values | get "teraslice.stateCluster" "none") "elasticsearch6" }} + - "http://elasticsearch6-master.services-dev1:9200" + {{- else if eq (.Values | get "teraslice.stateCluster" "none") "elasticsearch7" }} + - "http://elasticsearch7-master.services-dev1:9200" + {{- end }} + {{- if eq (.Values | get "opensearch1.enabled" false) true }} + os1: + node: + - "http://opensearch1.services-dev1:9200" + {{- end }} + {{- if eq (.Values | get "opensearch2.enabled" false) true }} + os2: + node: + - "http://opensearch2.services-dev1:9200" + {{- end }} + {{- if eq (.Values | get "elasticsearch6.enabled" false) true }} + es6: + node: + - "http://elasticsearch6-master.services-dev1:9200" + {{- end }} + {{- if eq (.Values | get "elasticsearch7.enabled" false) true }} + es7: + node: + - "http://elasticsearch7-master.services-dev1:9200" + {{- end }} + {{- if eq (.Values | get "kafka.enabled" false) true }} + kafka: + default: + brokers: + - "kafka-cp-kafka.services-dev1:9092" + {{- end }} + {{- if eq (.Values | get "minio.enabled" false) true }} + s3: + default: + endpoint: "http://minio.services-dev1:9000" + accessKeyId: "minioadmin" + secretAccessKey: "minioadmin" + forcePathStyle: true + sslEnabled: false + region: "us-east-1" + {{- end }} + +master: + teraslice: + kubernetes_namespace: ts-dev1 + cluster_manager_type: kubernetesV2 + asset_storage_connection_type: elasticsearch-next + name: "ts-dev1" + +worker: + teraslice: + kubernetes_namespace: ts-dev1 + cluster_manager_type: kubernetesV2 + asset_storage_connection_type: elasticsearch-next + name: "ts-dev1" diff --git a/e2e/helm/values.yaml b/e2e/helm/values.yaml new file mode 100644 index 00000000000..0c8174ae0dd --- /dev/null +++ b/e2e/helm/values.yaml @@ -0,0 +1,81 @@ +opensearch1: + # If false, opensearch1 will be excluded on helmfile sync + enabled: false + version: 1.3.11 + esJavaOpts: -Xmx512M -Xms512M + memoryLimit: 100Mi + persistentVolumeSize: 8Gi + # The number of replicas + instances: 1 + +opensearch2: + # If false, opensearch2 will be excluded on helmfile sync + enabled: true + version: 2.15.0 + esJavaOpts: -Xmx512M -Xms512M + memoryLimit: 100Mi + persistentVolumeSize: 8Gi + # The number of replicas + instances: 1 + +elasticsearch6: + # If false, elasticsearch6 will be excluded on helmfile sync + enabled: false + version: 6.8.6 + # Overrides default image. Useful when loading in arm image that es doesn't support + # image: elasticsearch + # The number of replicas + instances: 1 + esJavaOpts: -Xms512m -Xmx512m + memoryLimit: 1Gi + persistentVolumeSize: 8Gi + +elasticsearch7: + # If false, elasticsearch7 will be excluded on helmfile sync + enabled: false + # It's not recommended to deviate from es versions 7.9. The chart is tied to this specific version. + version: 7.9.3 + # The number of replicas + instances: 1 + esJavaOpts: -Xms512m -Xmx512m + # Es7 recommends at least a gig of memory. + memoryLimit: 1Gi + persistentVolumeSize: 8Gi + +minio: + # If false, minio will be excluded on helmfile sync + enabled: false + version: RELEASE.2024-08-29T01-40-52Z + instances: 1 + rootUser: minioadmin + rootPassword: minioadmin + memoryLimit: 256Mi + persistentVolumeSize: 50Gi + # tls: <-- Not implemented yet + # enabled: false + # certSecret: "" + # publicCrt: public.crt + # privateKey: private.key + +kafka: + # If false, kafka will be excluded on helmfile sync + enabled: true + version: 7.7.1 + brokers: 1 + offsets: + topic: + replication: + factor: 1 + +zookeeper: + # If false, zookeeper will be excluded on helmfile sync + enabled: true + version: 7.7.1 + instances: 1 + +teraslice: + image: + repository: teraslice-workspace + # Overrides the image tag whose default is the chart appVersion. + tag: e2e-nodev22 + stateCluster: opensearch2 diff --git a/e2e/k8s/kindConfigTestPorts.yaml b/e2e/k8s/kindConfigTestPorts.yaml index d0f183f9801..e76695946bf 100644 --- a/e2e/k8s/kindConfigTestPorts.yaml +++ b/e2e/k8s/kindConfigTestPorts.yaml @@ -5,10 +5,13 @@ nodes: - role: control-plane image: kindest/node:v1.28.12@sha256:fa0e48b1e83bb8688a5724aa7eebffbd6337abd7909ad089a2700bf08c30c6ea extraPortMappings: - - containerPort: 30200 # Map internal elasticsearch service to host port - hostPort: 49200 + # teraslice port mapping needs to be at index 0 in this array - containerPort: 30678 # Map internal teraslice service to host port hostPort: 45678 + - containerPort: 30200 # Map internal elasticsearch service to host port + hostPort: 49200 + - containerPort: 30210 # Map internal opensearch service to host port + hostPort: 49210 - containerPort: 30092 # Map internal kafka service to host port hostPort: 49092 - containerPort: 30094 # Map external kafka service to host port diff --git a/e2e/package.json b/e2e/package.json index 78403943352..bb4ffd3110d 100644 --- a/e2e/package.json +++ b/e2e/package.json @@ -1,7 +1,7 @@ { "name": "e2e", "displayName": "E2E Tests", - "version": "0.7.1", + "version": "0.8.0", "private": true, "description": "Teraslice integration test suite", "keywords": [ @@ -32,10 +32,12 @@ "test:elasticsearch6": "TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='6.8.6' TEST_KAFKA='true' ts-scripts test --suite e2e --", "test:elasticsearch7": "TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' TEST_KAFKA='true' ts-scripts test --suite e2e --", "test:k8s": "TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' TEST_KAFKA='true' TEST_PLATFORM='kubernetes' ts-scripts test --suite e2e --", + "test:k8sHelmfile": "TEST_OPENSEARCH='true' OPENSEARCH_VERSION='2.15.0' TEST_KAFKA='true' TEST_PLATFORM='kubernetes' USE_HELMFILE='true' ts-scripts test --suite e2e --", "test:k8sNoBuild": "SKIP_DOCKER_BUILD_IN_E2E='true' TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' TEST_KAFKA='true' TEST_PLATFORM='kubernetes' ts-scripts test --suite e2e --", "test:k8sV2": "TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' TEST_KAFKA='true' TEST_PLATFORM='kubernetesV2' ts-scripts test --suite e2e --", + "test:k8sV2Helmfile": "TEST_OPENSEARCH='true' OPENSEARCH_VERSION='2.15.0' TEST_KAFKA='true' TEST_PLATFORM='kubernetesV2' USE_HELMFILE='true' ts-scripts test --suite e2e --", "test:opensearch1": "TEST_OPENSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite e2e --", - "test:opensearch2": "TEST_OPENSEARCH='true' OPENSEARCH_VERSION='2.8.0' TEST_KAFKA='true' ts-scripts test --suite e2e --", + "test:opensearch2": "TEST_OPENSEARCH='true' OPENSEARCH_VERSION='2.15.0' TEST_KAFKA='true' ts-scripts test --suite e2e --", "test:s3AssetStorage": "TEST_OPENSEARCH='true' TEST_KAFKA='true' TEST_MINIO='true' ASSET_STORAGE_CONNECTION_TYPE='s3' ASSET_STORAGE_CONNECTION='default' ts-scripts test --suite e2e --", "test:watch": "TEST_ELASTICSEARCH='true' TEST_KAFKA='true' ts-scripts test --suite e2e --watch --" }, @@ -43,8 +45,9 @@ "ms": "~2.1.3" }, "devDependencies": { - "@terascope/scripts": "~1.9.3", + "@terascope/scripts": "~1.10.0", "@terascope/types": "~1.4.1", + "@terascope/utils": "~1.7.3", "bunyan": "~1.8.15", "elasticsearch-store": "~1.8.1", "fs-extra": "~11.3.0", diff --git a/e2e/test/config.ts b/e2e/test/config.ts index cf94021a70f..a9808c72ed6 100644 --- a/e2e/test/config.ts +++ b/e2e/test/config.ts @@ -2,6 +2,7 @@ import { ElasticsearchTestHelpers } from 'elasticsearch-store'; import { customAlphabet } from 'nanoid'; import { fileURLToPath } from 'node:url'; import path from 'node:path'; +import { toBoolean } from '@terascope/utils'; const { TEST_INDEX_PREFIX, @@ -60,6 +61,7 @@ const { } = process.env; const TEST_HOST = TEST_OPENSEARCH ? OPENSEARCH_HOST : ELASTICSEARCH_HOST; +const USE_HELMFILE = toBoolean(process.env.USE_HELMFILE) || false; function newId(prefix?: string, lowerCase = false, length = 15) { let characters = '0123456789abcdefghijklmnopqrstuvwxyz'; @@ -112,5 +114,6 @@ export { MINIO_SECRET_KEY, ENCRYPT_MINIO, ROOT_CERT_PATH, - TEST_OPENSEARCH + TEST_OPENSEARCH, + USE_HELMFILE }; diff --git a/e2e/test/global.setup.ts b/e2e/test/global.setup.ts index 90dae88b24f..5578e1456d0 100644 --- a/e2e/test/global.setup.ts +++ b/e2e/test/global.setup.ts @@ -1,6 +1,6 @@ import { pDelay } from '@terascope/utils'; import fse from 'fs-extra'; -import { K8s, setAlias } from '@terascope/scripts'; +import { helmfileSync, K8s, setAlias } from '@terascope/scripts'; import { TerasliceHarness } from './teraslice-harness.js'; import { dockerUp } from './docker-helpers.js'; import signale from './signale.js'; @@ -8,7 +8,7 @@ import setupTerasliceConfig from './setup-config.js'; import { downloadAssets } from './download-assets.js'; import { CONFIG_PATH, ASSETS_PATH, TEST_PLATFORM, - TERASLICE_PORT, KIND_CLUSTER + TERASLICE_PORT, KIND_CLUSTER, USE_HELMFILE } from './config.js'; import { teardown } from './teardown.js'; @@ -36,8 +36,12 @@ export default async () => { if (TEST_PLATFORM === 'kubernetes' || TEST_PLATFORM === 'kubernetesV2') { await downloadAssets(); - const k8s = new K8s(TERASLICE_PORT, KIND_CLUSTER); - await k8s.deployK8sTeraslice(TEST_PLATFORM, true, false); + if (USE_HELMFILE) { + await helmfileSync(); + } else { + const k8s = new K8s(TERASLICE_PORT, KIND_CLUSTER); + await k8s.deployK8sTeraslice(TEST_PLATFORM, true, false); + } await teraslice.waitForTeraslice(); await setAlias(TERASLICE_PORT); } else { diff --git a/e2e/test/teardown.ts b/e2e/test/teardown.ts index 05566a836a6..f336211e3ef 100644 --- a/e2e/test/teardown.ts +++ b/e2e/test/teardown.ts @@ -1,9 +1,9 @@ import { ElasticsearchTestHelpers, Client } from 'elasticsearch-store'; -import { K8s } from '@terascope/scripts'; +import { helmfileDelete, K8s } from '@terascope/scripts'; import fse from 'fs-extra'; import { - KEEP_OPEN, CONFIG_PATH, ASSETS_PATH, - TEST_INDEX_PREFIX, TEST_PLATFORM, TERASLICE_PORT, KIND_CLUSTER + KEEP_OPEN, CONFIG_PATH, ASSETS_PATH, TEST_INDEX_PREFIX, + TEST_PLATFORM, TERASLICE_PORT, KIND_CLUSTER, USE_HELMFILE } from './config.js'; import { tearDown } from './docker-helpers.js'; import signale from './signale.js'; @@ -26,8 +26,12 @@ export async function teardown(testClient?: Client) { try { if (TEST_PLATFORM === 'kubernetes' || TEST_PLATFORM === 'kubernetesV2') { - const k8s = new K8s(TERASLICE_PORT, KIND_CLUSTER); - await k8s.deleteTerasliceNamespace('ts-ns.yaml'); + if (USE_HELMFILE) { + await helmfileDelete('teraslice'); + } else { + const k8s = new K8s(TERASLICE_PORT, KIND_CLUSTER); + await k8s.deleteTerasliceNamespace('ts-ns.yaml'); + } await cleanupIndex(client, 'ts-dev1_*'); } else { await tearDown(); diff --git a/helm/teraslice/templates/deployment.yaml b/helm/teraslice/templates/deployment.yaml index 2f665304b88..173bdcb1352 100644 --- a/helm/teraslice/templates/deployment.yaml +++ b/helm/teraslice/templates/deployment.yaml @@ -70,6 +70,9 @@ spec: mountPath: /app/config - name: teraslice-assets mountPath: /app/assets + - name: autoload + mountPath: /app/autoload + readOnly: true {{- range .Values.dataVolumes }} - name: {{ .name }} mountPath: {{ .path }} @@ -122,6 +125,10 @@ spec: {{- else }} emptyDir: {} {{- end }} + - name: autoload + hostPath: + path: /autoload + type: Directory {{- range .Values.dataVolumes }} - name: {{ .name }} persistentVolumeClaim: diff --git a/helm/teraslice/values.yaml b/helm/teraslice/values.yaml index ad624a2bc67..3af4f3ac1fc 100644 --- a/helm/teraslice/values.yaml +++ b/helm/teraslice/values.yaml @@ -38,12 +38,14 @@ priorityClass: master: teraslice: assets_directory: /app/assets + autoload_directory: /app/autoload workers: 0 # Teraslice Worker Config Options worker: teraslice: assets_directory: /app/assets + autoload_directory: /app/autoload # Terafoundation # Everything under terafoundation key will used as the master and worker config diff --git a/package.json b/package.json index ca759460388..4304cc15172 100644 --- a/package.json +++ b/package.json @@ -53,7 +53,7 @@ "@eslint/js": "~9.19.0", "@swc/core": "1.10.9", "@swc/jest": "~0.2.37", - "@terascope/scripts": "~1.9.3", + "@terascope/scripts": "~1.10.0", "@types/bluebird": "~3.5.42", "@types/convict": "~6.1.6", "@types/elasticsearch": "~5.0.43", diff --git a/packages/elasticsearch-api/package.json b/packages/elasticsearch-api/package.json index 9b6153eee41..82ce93e916f 100644 --- a/packages/elasticsearch-api/package.json +++ b/packages/elasticsearch-api/package.json @@ -20,7 +20,7 @@ "test:elasticsearch7": "TEST_RESTRAINED_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' yarn workspace @terascope/scripts ts-scripts test ../elasticsearch-api --", "test:elasticsearch8": "TEST_RESTRAINED_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='8.1.2' yarn workspace @terascope/scripts ts-scripts test ../elasticsearch-api --", "test:opensearch1": "TEST_RESTRAINED_OPENSEARCH='true' yarn workspace @terascope/scripts ts-scripts test --debug ../elasticsearch-api --", - "test:opensearch2": "TEST_RESTRAINED_OPENSEARCH='true' OPENSEARCH_VERSION='2.8.0' yarn workspace @terascope/scripts ts-scripts test --debug ../elasticsearch-api --", + "test:opensearch2": "TEST_RESTRAINED_OPENSEARCH='true' OPENSEARCH_VERSION='2.15.0' yarn workspace @terascope/scripts ts-scripts test --debug ../elasticsearch-api --", "test:watch": "TEST_RESTRAINED_ELASTICSEARCH='true' yarn workspace @terascope/scripts ts-scripts test --watch ../elasticsearch-api --" }, "dependencies": { diff --git a/packages/elasticsearch-store/package.json b/packages/elasticsearch-store/package.json index ba96ad8bc0b..223f28caea9 100644 --- a/packages/elasticsearch-store/package.json +++ b/packages/elasticsearch-store/package.json @@ -26,7 +26,7 @@ "test:elasticsearch7": "TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' yarn workspace @terascope/scripts ts-scripts test ../elasticsearch-store --", "test:elasticsearch8": "TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='8.1.2' yarn workspace @terascope/scripts ts-scripts test ../elasticsearch-store --", "test:opensearch1": "TEST_OPENSEARCH='true' yarn workspace @terascope/scripts ts-scripts test ../elasticsearch-store --", - "test:opensearch2": "TEST_OPENSEARCH='true' OPENSEARCH_VERSION='2.8.0' yarn workspace @terascope/scripts ts-scripts test ../elasticsearch-store --", + "test:opensearch2": "TEST_OPENSEARCH='true' OPENSEARCH_VERSION='2.15.0' yarn workspace @terascope/scripts ts-scripts test ../elasticsearch-store --", "test:watch": "ts-scripts yarn workspace @terascope/scripts test --watch ../elasticsearch-store --" }, "dependencies": { diff --git a/packages/scripts/package.json b/packages/scripts/package.json index c8968a99751..4afaaae564c 100644 --- a/packages/scripts/package.json +++ b/packages/scripts/package.json @@ -1,7 +1,7 @@ { "name": "@terascope/scripts", "displayName": "Scripts", - "version": "1.9.3", + "version": "1.10.0", "description": "A collection of terascope monorepo scripts", "homepage": "https://github.com/terascope/teraslice/tree/master/packages/scripts#readme", "bugs": { diff --git a/packages/scripts/src/cmds/test.ts b/packages/scripts/src/cmds/test.ts index bb22d88d965..c6768f62535 100644 --- a/packages/scripts/src/cmds/test.ts +++ b/packages/scripts/src/cmds/test.ts @@ -23,6 +23,7 @@ type Options = { 'ignore-mount': boolean; 'test-platform': string; 'skip-image-deletion': boolean; + 'use-helmfile': boolean; }; const jestArgs = getExtraArgs(); @@ -107,6 +108,11 @@ const cmd: CommandModule = { type: 'boolean', default: config.SKIP_IMAGE_DELETION, }) + .option('use-helmfile', { + description: 'If true k8s tests will launch using helmfile, if false tests use kubectl and @kubernetes/client-node', + type: 'boolean', + default: config.USE_HELMFILE, + }) .positional('packages', { description: 'Runs the tests for one or more package and/or an asset, if none specified it will run all of the tests', coerce(arg) { @@ -133,6 +139,7 @@ const cmd: CommandModule = { const testPlatform = hoistJestArg(argv, 'test-platform', 'string') as 'native' | 'kubernetes' | 'kubernetesV2'; const kindClusterName = testPlatform === 'native' ? 'default' : 'k8s-e2e'; const skipImageDeletion = hoistJestArg(argv, 'skip-image-deletion', 'boolean'); + const useHelmfile = hoistJestArg(argv, 'use-helmfile', 'boolean'); if (debug && watch) { throw new Error('--debug and --watch conflict, please set one or the other'); @@ -154,7 +161,8 @@ const cmd: CommandModule = { ignoreMount, testPlatform, kindClusterName, - skipImageDeletion + skipImageDeletion, + useHelmfile }); }, }; diff --git a/packages/scripts/src/helpers/config.ts b/packages/scripts/src/helpers/config.ts index df35f1a86cb..a36a00c724a 100644 --- a/packages/scripts/src/helpers/config.ts +++ b/packages/scripts/src/helpers/config.ts @@ -19,9 +19,9 @@ export const __DEFAULT_ELASTICSEARCH6_VERSION = '6.8.6'; /** Default elasticsearch7 version used to populate the CI cache */ export const __DEFAULT_ELASTICSEARCH7_VERSION = '7.9.3'; /** Default opensearch1 version used to populate the CI cache */ -export const __DEFAULT_OPENSEARCH1_VERSION = '1.3.10'; +export const __DEFAULT_OPENSEARCH1_VERSION = '1.3.11'; /** Default opensearch2 version used to populate the CI cache */ -export const __DEFAULT_OPENSEARCH2_VERSION = '2.8.0'; +export const __DEFAULT_OPENSEARCH2_VERSION = '2.15.0'; export const TERASLICE_PORT = '45678'; export const HOST_IP = process.env.HOST_IP || address(); @@ -46,7 +46,7 @@ export const KAFKA_NAME = process.env.KAFKA_NAME || 'kafka'; export const KAFKA_HOSTNAME = process.env.KAFKA_HOSTNAME || HOST_IP; export const KAFKA_PORT = process.env.KAFKA_PORT || '49092'; export const KAFKA_BROKER = `${KAFKA_HOSTNAME}:${KAFKA_PORT}`; -export const KAFKA_VERSION = process.env.KAFKA_VERSION || '3.5'; +export const KAFKA_VERSION = process.env.KAFKA_VERSION || '3.7'; // Use kafkaVersionMapper to determine confluentinc/cp-kafka image version from KAFKA_VERSION export const KAFKA_IMAGE_VERSION = kafkaVersionMapper(KAFKA_VERSION); export const KAFKA_DOCKER_IMAGE = process.env.KAFKA_DOCKER_IMAGE || 'confluentinc/cp-kafka'; @@ -207,3 +207,4 @@ export const DOCKER_IMAGES_PATH = './images'; export const DOCKER_IMAGE_LIST_PATH = `${DOCKER_IMAGES_PATH}/image-list.txt`; export const DOCKER_CACHE_PATH = '/tmp/docker_cache'; export const SKIP_IMAGE_DELETION = toBoolean(process.env.SKIP_IMAGE_DELETION) || false; +export const USE_HELMFILE = toBoolean(process.env.USE_HELMFILE) || false; diff --git a/packages/scripts/src/helpers/k8s-env/index.ts b/packages/scripts/src/helpers/k8s-env/index.ts index d36b34e8e8e..be29e795db4 100644 --- a/packages/scripts/src/helpers/k8s-env/index.ts +++ b/packages/scripts/src/helpers/k8s-env/index.ts @@ -107,7 +107,8 @@ export async function launchK8sEnv(options: K8sEnvOptions) { useExistingServices: false, ignoreMount: false, testPlatform: options.clusteringType, - skipImageDeletion: false + skipImageDeletion: false, + useHelmfile: false }); try { diff --git a/packages/scripts/src/helpers/kind.ts b/packages/scripts/src/helpers/kind.ts index ee790a0fa7e..84d9f600d91 100644 --- a/packages/scripts/src/helpers/kind.ts +++ b/packages/scripts/src/helpers/kind.ts @@ -54,7 +54,7 @@ export class Kind { configFile.nodes[0].extraMounts.push(...dockerFileMounts); } } - configFile.nodes[0].extraPortMappings[1].hostPort = Number.parseInt(teraslicePort, 10); + configFile.nodes[0].extraPortMappings[0].hostPort = Number.parseInt(teraslicePort, 10); const updatedYaml = yaml.dump(configFile); const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'tempYaml')); diff --git a/packages/scripts/src/helpers/mapper.ts b/packages/scripts/src/helpers/mapper.ts index 497b6d74b0f..faabebecedd 100644 --- a/packages/scripts/src/helpers/mapper.ts +++ b/packages/scripts/src/helpers/mapper.ts @@ -11,19 +11,22 @@ export function kafkaVersionMapper(kafkaVersion: string): string { } const kafkaMapper: Record> = { 3: { - 0: '7.0.11', - 1: '7.1.9', - 2: '7.2.7', - 3: '7.3.5', - 4: '7.4.2', - 5: '7.5.1' + 0: '7.0.16', + 1: '7.1.15', + 2: '7.2.13', + 3: '7.3.11', + 4: '7.4.8', + 5: '7.5.7', + 6: '7.6.4', + 7: '7.7.2', + 8: '7.8.0' }, 2: { 4: '5.4.10', 5: '5.5.12', 6: '6.0.15', - 7: '6.1.13', - 8: '6.2.12' + 7: '6.1.15', + 8: '6.2.15' } }; diff --git a/packages/scripts/src/helpers/scripts.ts b/packages/scripts/src/helpers/scripts.ts index 8000eeb28de..44c6e2acd12 100644 --- a/packages/scripts/src/helpers/scripts.ts +++ b/packages/scripts/src/helpers/scripts.ts @@ -13,9 +13,10 @@ import { TSCommands, PackageInfo } from './interfaces.js'; import { getRootDir } from './misc.js'; import signale from './signale.js'; import * as config from './config.js'; -import { getE2eK8sDir } from '../helpers/packages.js'; +import { getE2EDir, getE2eK8sDir } from '../helpers/packages.js'; import { YamlDeploymentResource, YamlServiceResource } from './k8s-env/interfaces.js'; import { Kind } from './kind.js'; +import { ENV_SERVICES } from './config.js'; const logger = debugLogger('ts-scripts:cmd'); @@ -625,6 +626,24 @@ export async function isKubectlInstalled(): Promise { } } +export async function isHelmInstalled(): Promise { + try { + const subprocess = await execaCommand('command -v helm'); + return !!subprocess.stdout; + } catch (err) { + return false; + } +} + +export async function isHelmfileInstalled(): Promise { + try { + const subprocess = await execaCommand('command -v helmfile'); + return !!subprocess.stdout; + } catch (err) { + return false; + } +} + export async function k8sStopService(serviceName: string): Promise { const e2eK8sDir = getE2eK8sDir(); if (!e2eK8sDir) { @@ -730,7 +749,7 @@ export async function showState(tsPort: string) { } async function showESIndices() { - const subprocess = await execaCommand(`curl ${config.HOST_IP}:${config.ELASTICSEARCH_PORT}/_cat/indices?v`); + const subprocess = await execaCommand(`curl ${config.SEARCH_TEST_HOST}/_cat/indices?v`); return subprocess.stdout; } @@ -762,3 +781,90 @@ export async function logTCPPorts() { signale.error('Execa command failed trying to log ports: ', err); } } + +export async function helmfileDelete(selector: string) { + const e2eDir = getE2EDir(); + if (!e2eDir) { + throw new Error('Missing e2e test directory'); + } + const helmfilePath = path.join(e2eDir, 'helm/helmfile.yaml'); + + try { + const subprocess = await execaCommand(`helmfile delete -f ${helmfilePath} --selector app=${selector}`); + logger.debug('helmfile delete: ', subprocess.stdout); + } catch (err) { + logger.info(err); + } +} + +export async function helmfileDiff() { + const e2eDir = getE2EDir(); + if (!e2eDir) { + throw new Error('Missing e2e test directory'); + } + const helmfilePath = path.join(e2eDir, 'helm/helmfile.yaml'); + const values = createValuesStringFromServicesArray(); + + const subprocess = await execaCommand(`helmfile ${values} diff -f ${helmfilePath} --suppress-secrets`); + logger.debug('helmfile diff: ', subprocess.stdout); +} + +export async function helmfileSync() { + const e2eDir = getE2EDir(); + if (!e2eDir) { + throw new Error('Missing e2e test directory'); + } + const helmfilePath = path.join(e2eDir, 'helm/helmfile.yaml'); + const values = createValuesStringFromServicesArray(); + + const subprocess = await execaCommand(`helmfile ${values} sync -f ${helmfilePath}`); + logger.debug('helmfile sync: ', subprocess.stdout); +} + +export async function launchE2EWithHelmfile() { + await helmfileDiff(); + await helmfileSync(); +} + +function createValuesStringFromServicesArray() { + let values = ENV_SERVICES.reduce((valuesString, service) => { + let serviceString = service.toString(); + let version; + let stateCluster; + let newValuesString; + + // Setting the stateCluster will only work properly if there is a single ES/OS service + // If we need multiple in the future we need to modify this. + if (service === 'opensearch') { + serviceString += config.OPENSEARCH_VERSION.charAt(0); + version = config.OPENSEARCH_VERSION; + stateCluster = serviceString; + } + if (service === 'elasticsearch') { + serviceString += config.ELASTICSEARCH_VERSION.charAt(0); + version = config.ELASTICSEARCH_VERSION; + stateCluster = serviceString; + } + if (service === 'kafka') { + version = config.KAFKA_IMAGE_VERSION; + } + if (service === 'zookeeper') { + version = config.ZOOKEEPER_VERSION; + } + if (service === 'minio') { + version = config.MINIO_VERSION; + } + + newValuesString = `${valuesString} --state-values-set ${serviceString}.enabled=true --state-values-set ${serviceString}.version=${version}`; + + if (stateCluster) { + newValuesString += ` --state-values-set teraslice.stateCluster=${stateCluster}`; + } + + return newValuesString; + }, ''); + + values += ` --state-values-set teraslice.image.tag=e2e-nodev${config.NODE_VERSION}`; + logger.debug('helmfile command values: ', values); + return values; +} diff --git a/packages/scripts/src/helpers/test-runner/index.ts b/packages/scripts/src/helpers/test-runner/index.ts index 9f871934a07..8f5a89675b1 100644 --- a/packages/scripts/src/helpers/test-runner/index.ts +++ b/packages/scripts/src/helpers/test-runner/index.ts @@ -10,9 +10,9 @@ import { ensureServices, loadOrPullServiceImages } from './services.js'; import { PackageInfo } from '../interfaces.js'; import { TestOptions } from './interfaces.js'; import { - runJest, dockerTag, isKindInstalled, - isKubectlInstalled, loadThenDeleteImageFromCache, - deleteDockerImageCache + runJest, dockerTag, isKindInstalled, isKubectlInstalled, + loadThenDeleteImageFromCache, deleteDockerImageCache, + isHelmInstalled, isHelmfileInstalled, launchE2EWithHelmfile } from '../scripts.js'; import { Kind } from '../kind.js'; import { @@ -212,7 +212,19 @@ async function runE2ETest( const kubectlInstalled = await isKubectlInstalled(); if (!kubectlInstalled && !isCI) { - signale.error('Please install kubectl before running k8s tests. https://kubernetes.io/docs/tasks/tools/'); + signale.error('Please install kubectl before running k8s tests. https://kubernetes.io/docs/tasks/tools'); + process.exit(1); + } + + const helmInstalled = await isHelmInstalled(); + if (!helmInstalled && !isCI) { + signale.error('Please install Helm before running k8s tests.https://helm.sh/docs/intro/install'); + process.exit(1); + } + + const helmfileInstalled = await isHelmfileInstalled(); + if (!helmfileInstalled && !isCI) { + signale.error('Please install helmfile before running k8s tests. https://helmfile.readthedocs.io/en/latest/#installation'); process.exit(1); } @@ -227,8 +239,10 @@ async function runE2ETest( await kind.destroyCluster(); process.exit(1); } - const k8s = new K8s(TERASLICE_PORT, options.kindClusterName); - await k8s.createNamespace('services-ns.yaml', 'services'); + if (!options.useHelmfile) { + const k8s = new K8s(TERASLICE_PORT, options.kindClusterName); + await k8s.createNamespace('services-ns.yaml', 'services'); + } } catch (err) { tracker.addError(err); } @@ -268,18 +282,26 @@ async function runE2ETest( if (kind && (options.testPlatform === 'kubernetes' || options.testPlatform === 'kubernetesV2')) { try { await kind.loadTerasliceImage(e2eImage); + if (options.useHelmfile) { + const timeLabel = 'helmfile deployment'; + signale.time(timeLabel); + await launchE2EWithHelmfile(); + signale.timeEnd(timeLabel); + } } catch (err) { tracker.addError(err); } } - try { - tracker.addCleanup( - 'e2e:services', - await ensureServices(suite, options) - ); - } catch (err) { - tracker.addError(err); + if (!options.useHelmfile) { + try { + tracker.addCleanup( + 'e2e:services', + await ensureServices(suite, options) + ); + } catch (err) { + tracker.addError(err); + } } if (!options.skipImageDeletion) { diff --git a/packages/scripts/src/helpers/test-runner/interfaces.ts b/packages/scripts/src/helpers/test-runner/interfaces.ts index e886964baf6..8bfdeb44a95 100644 --- a/packages/scripts/src/helpers/test-runner/interfaces.ts +++ b/packages/scripts/src/helpers/test-runner/interfaces.ts @@ -17,6 +17,7 @@ export type TestOptions = { testPlatform: 'native' | 'kubernetes' | 'kubernetesV2'; kindClusterName: string; skipImageDeletion: boolean; + useHelmfile: boolean; }; export type GroupedPackages = { diff --git a/packages/scripts/src/helpers/test-runner/utils.ts b/packages/scripts/src/helpers/test-runner/utils.ts index f1780913789..f2a7430a8e1 100644 --- a/packages/scripts/src/helpers/test-runner/utils.ts +++ b/packages/scripts/src/helpers/test-runner/utils.ts @@ -71,7 +71,8 @@ export function getEnv(options: TestOptions, suite: string): ExecEnv { KIND_CLUSTER: options.kindClusterName, TERASLICE_PORT: config.TERASLICE_PORT, TJM_TEST_MODE: suite !== 'e2e' ? 'true' : 'false', - NODE_OPTIONS: '--experimental-vm-modules' + NODE_OPTIONS: '--experimental-vm-modules', + USE_HELMFILE: options.useHelmfile ? 'true' : 'false' }; if (config.DOCKER_NETWORK_NAME) { @@ -138,7 +139,6 @@ export function getEnv(options: TestOptions, suite: string): ExecEnv { DISABLE_SECURITY_PLUGIN: true, DISABLE_INSTALL_DEMO_CONFIG: true, SEARCH_TEST_HOST: `${config.SEARCH_TEST_HOST}` - }); } diff --git a/packages/scripts/test/helpers-spec.ts b/packages/scripts/test/helpers-spec.ts index 403a4431c61..5fe37f5f648 100644 --- a/packages/scripts/test/helpers-spec.ts +++ b/packages/scripts/test/helpers-spec.ts @@ -49,15 +49,15 @@ describe('Helpers', () => { expect(() => kafkaVersionMapper('2.3')).toThrow(); expect(() => kafkaVersionMapper('2.9')).toThrow(); expect(() => kafkaVersionMapper('3')).toThrow(); - expect(() => kafkaVersionMapper('3.6')).toThrow(); + expect(() => kafkaVersionMapper('3.9')).toThrow(); expect(() => kafkaVersionMapper('4.0')).toThrow(); }); it('should be able to convert kafka versions to the proper confluent/cp-kafka versions', () => { - expect(kafkaVersionMapper('3.1')).toBe('7.1.9'); - expect(kafkaVersionMapper('3.5')).toBe('7.5.1'); + expect(kafkaVersionMapper('3.1')).toBe('7.1.15'); + expect(kafkaVersionMapper('3.5')).toBe('7.5.7'); expect(kafkaVersionMapper('2.4')).toBe('5.4.10'); - expect(kafkaVersionMapper('2.8')).toBe('6.2.12'); + expect(kafkaVersionMapper('2.8')).toBe('6.2.15'); }); }); }); diff --git a/packages/scripts/test/service-spec.ts b/packages/scripts/test/service-spec.ts index e6469d1b7a0..c2fdc2cb4c3 100644 --- a/packages/scripts/test/service-spec.ts +++ b/packages/scripts/test/service-spec.ts @@ -111,7 +111,8 @@ describe('services', () => { ignoreMount: false, testPlatform: 'native', kindClusterName: 'default', - skipImageDeletion: false + skipImageDeletion: false, + useHelmfile: false }; let services: any; diff --git a/packages/scripts/test/test-runner-spec.ts b/packages/scripts/test/test-runner-spec.ts index d428592aaef..82f90e0735f 100644 --- a/packages/scripts/test/test-runner-spec.ts +++ b/packages/scripts/test/test-runner-spec.ts @@ -22,7 +22,8 @@ describe('Test Runner Helpers', () => { ignoreMount: true, testPlatform: 'native', kindClusterName: 'default', - skipImageDeletion: false + skipImageDeletion: false, + useHelmfile: false }; function makeTestOptions(input: Partial): TestOptions { diff --git a/packages/teraslice/package.json b/packages/teraslice/package.json index 02e7cec371d..88205c7c864 100644 --- a/packages/teraslice/package.json +++ b/packages/teraslice/package.json @@ -30,7 +30,7 @@ "test:elasticsearch6": "TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='6.8.6' yarn workspace @terascope/scripts ts-scripts test ../teraslice --", "test:elasticsearch7": "TEST_ELASTICSEARCH='true' ELASTICSEARCH_VERSION='7.9.3' yarn workspace @terascope/scripts ts-scripts test ../teraslice --", "test:opensearch1": "TEST_OPENSEARCH='true' yarn workspace @terascope/scripts ts-scripts test ../teraslice --", - "test:opensearch2": "TEST_OPENSEARCH='true' OPENSEARCH_VERSION='2.8.0' yarn workspace @terascope/scripts ts-scripts test ../teraslice --", + "test:opensearch2": "TEST_OPENSEARCH='true' OPENSEARCH_VERSION='2.15.0' yarn workspace @terascope/scripts ts-scripts test ../teraslice --", "test:watch": "TEST_ELASTICSEARCH='true' yarn workspace @terascope/scripts ts-scripts test --watch ../teraslice --" }, "resolutions": { diff --git a/yarn.lock b/yarn.lock index dc6020fafb8..dbaae941935 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2964,7 +2964,7 @@ __metadata: languageName: unknown linkType: soft -"@terascope/scripts@npm:~1.9.3, @terascope/scripts@workspace:packages/scripts": +"@terascope/scripts@npm:~1.10.0, @terascope/scripts@workspace:packages/scripts": version: 0.0.0-use.local resolution: "@terascope/scripts@workspace:packages/scripts" dependencies: @@ -6415,8 +6415,9 @@ __metadata: version: 0.0.0-use.local resolution: "e2e@workspace:e2e" dependencies: - "@terascope/scripts": "npm:~1.9.3" + "@terascope/scripts": "npm:~1.10.0" "@terascope/types": "npm:~1.4.1" + "@terascope/utils": "npm:~1.7.3" bunyan: "npm:~1.8.15" elasticsearch-store: "npm:~1.8.1" fs-extra: "npm:~11.3.0" @@ -13387,7 +13388,7 @@ __metadata: "@eslint/js": "npm:~9.19.0" "@swc/core": "npm:1.10.9" "@swc/jest": "npm:~0.2.37" - "@terascope/scripts": "npm:~1.9.3" + "@terascope/scripts": "npm:~1.10.0" "@types/bluebird": "npm:~3.5.42" "@types/convict": "npm:~6.1.6" "@types/elasticsearch": "npm:~5.0.43"