[e2e, scripts] Start K8s e2e from helmfile (#3929)

This PR makes the following changes: - creates a `helm` directory in `e2e` with the following: - helmfile.yaml - values.yaml - modified elastic/elasticsearch-6.8.9 chart - modified elastic/elasticsearch-7.9.3 chart - modified confluentinc/cp-helm-charts-0.6.1 chart (kafka and zookeeper) - go templates for teraslice and services - update the teraslice helm chart to include the `autoload_directory` - update `ts-scripts test` to start k8s e2e tests via helmfile using the -use-helmfile` flag - update e2e to use helmfile commands instead of the k8s api when started via helm - update default service versions for tests - kafka version from `3.5` to `3.7`, which is `cp-kafka:v7.7.2` - opensearch1 from `1.3.10` to `1.3.11` - opensearch2 from `2.8.0` to `2.15.0` - k8s e2e tests now run on opensearch2, not elasticsearch7 - update mappings between kafka and cp-kafka versions - bump e2e version from `0.7.1` to `0.8.0` - bump scripts version from `1.9.3` to `1.10.0` ref: #3931 --------- Co-authored-by: Joseph Soto <jsoto@terascope.com>
terascope · Jan 31, 2025 · 10882d9 · 10882d9
1 parent aaa27c4
commit 10882d9
Show file tree

Hide file tree

Showing 40 changed files with 791 additions and 155 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -74,15 +74,14 @@ jobs:
       - name: Test
         run: yarn test --suite unit --
 
-  teraslice-elasticsearch-tests:
+  e2e-k8s-tests:
     runs-on: ubuntu-latest
     needs: [compute-node-version-vars, verify-build, cache-docker-images]
     strategy:
       # opensearch is finiky, keep testing others if it fails
       fail-fast: false
       matrix:
-        node-version: ${{ fromJSON(needs.compute-node-version-vars.outputs.NODE_VERSIONS) }}
-        search-version: [elasticsearch6, elasticsearch7, opensearch1, opensearch2]
+        node-version: ${{ fromJSON(needs.compute-node-version-vars.outputs.NODE_VERSIONS_EXT_K8S) }}
     steps:
       - name: Check out code
         uses: actions/checkout@v4
@@ -115,19 +114,27 @@ jobs:
           path: /tmp/docker_cache
           key: docker-images-${{ hashFiles('./images/image-list.txt') }}
 
-      - name: Test ${{ matrix.search-version }}
-        run: yarn test:${{ matrix.search-version }}
-        working-directory: ./packages/teraslice
+      - name: Compile e2e code
+        run: yarn build
+        working-directory: ./e2e
 
-  elasticsearch-store-tests:
+      - name: Install Kind and Kubectl
+        uses: helm/kind-action@v1.10.0
+        with:
+          install_only: "true"
+
+      - name: Test k8s elasticsearch7
+        run: NODE_VERSION=${{ matrix.node-version }} yarn test:k8s
+        working-directory: ./e2e
+
+  e2e-k8s-v2-tests:
     runs-on: ubuntu-latest
     needs: [compute-node-version-vars, verify-build, cache-docker-images]
     strategy:
       # opensearch is finiky, keep testing others if it fails
       fail-fast: false
       matrix:
-        node-version: ${{ fromJSON(needs.compute-node-version-vars.outputs.NODE_VERSIONS) }}
-        search-version: [elasticsearch6, elasticsearch7, opensearch1, opensearch2]
+        node-version: ${{ fromJSON(needs.compute-node-version-vars.outputs.NODE_VERSIONS_EXT_K8S) }}
     steps:
       - name: Check out code
         uses: actions/checkout@v4
@@ -160,43 +167,46 @@ jobs:
           path: /tmp/docker_cache
           key: docker-images-${{ hashFiles('./images/image-list.txt') }}
 
-      - name: Test ${{ matrix.search-version }}
-        run: yarn test:${{ matrix.search-version }}
-        working-directory: ./packages/elasticsearch-store
-
-  lint-and-sync:
-    runs-on: ubuntu-latest
-    needs: [compute-node-version-vars, verify-build, cache-docker-images]
-    # will remove the checkout, build and setup when the artifact is made to just
-    # test the linting and syncing of the codebase
-    steps:
-    - name: Check out code
-      uses: actions/checkout@v4
-
-    - name: Setup Node
-      uses: actions/setup-node@v4
-      with:
-        node-version: ${{ fromJSON(needs.compute-node-version-vars.outputs.NODE_VERSION_MAIN) }}
-        cache: 'yarn'
+      - name: Compile e2e code
+        run: yarn build
+        working-directory: ./e2e
 
-    - name: Install and build packages
-      run: yarn && yarn setup
+      - name: Install Kind and Kubectl
+        uses: helm/kind-action@v1.10.0
+        with:
+          install_only: "true"
 
-    - name: Lint codebase
-      run: yarn lint
+      - name: Install Helmfile
+        run: |
+          VERSION=$(curl -s https://api.github.com/repos/helmfile/helmfile/releases/latest | jq -r '.tag_name')
+          VERSION_NO_V=${VERSION#v}
+          curl -L "https://github.com/helmfile/helmfile/releases/download/${VERSION}/helmfile_${VERSION_NO_V}_linux_amd64.tar.gz" -o helmfile.tgz
+          tar -xzf helmfile.tgz
+          chmod +x helmfile
+          mv helmfile /usr/local/bin/helmfile
+          helm plugin install https://github.com/databus23/helm-diff
+          helmfile version
+          helm version
+
+      - name: Test CLI Commands
+        run: |
+          command -v kind
+          command -v kubectl
+          command -v helm
+          command -v helmfile
 
-    - name: Sync codebase
-      run: yarn sync --verify
+      - name: Test k8s V2 opensearch2
+        run: NODE_VERSION=${{ matrix.node-version }} yarn test:k8sV2Helmfile
+        working-directory: ./e2e
 
-  elasticsearch-api-tests:
+  e2e-external-storage-tests:
     runs-on: ubuntu-latest
     needs: [compute-node-version-vars, verify-build, cache-docker-images]
     strategy:
       # opensearch is finiky, keep testing others if it fails
       fail-fast: false
       matrix:
-        node-version: ${{ fromJSON(needs.compute-node-version-vars.outputs.NODE_VERSIONS) }}
-        search-version: [elasticsearch6, elasticsearch7, opensearch1, opensearch2]
+        node-version: ${{ fromJSON(needs.compute-node-version-vars.outputs.NODE_VERSIONS_EXT_STORAGE) }}
     steps:
       - name: Check out code
         uses: actions/checkout@v4
@@ -229,19 +239,22 @@ jobs:
           path: /tmp/docker_cache
           key: docker-images-${{ hashFiles('./images/image-list.txt') }}
 
-      - name: Test ${{ matrix.search-version }}
-        run: yarn test:${{ matrix.search-version }}
-        working-directory: ./packages/elasticsearch-api
+      - name: Compile e2e code
+        run: yarn build
+        working-directory: ./e2e
 
-  e2e-tests:
+      - name: Test external Asset Storage opensearch1
+        run: NODE_VERSION=${{ matrix.node-version }} yarn test:s3AssetStorage
+        working-directory: ./e2e
+
+  e2e-external-storage-tests-encrypted:
     runs-on: ubuntu-latest
     needs: [compute-node-version-vars, verify-build, cache-docker-images]
     strategy:
       # opensearch is finiky, keep testing others if it fails
       fail-fast: false
       matrix:
-        node-version: ${{ fromJSON(needs.compute-node-version-vars.outputs.NODE_VERSIONS) }}
-        search-version: [elasticsearch6, elasticsearch7, opensearch1, opensearch2]
+        node-version: ${{ fromJSON(needs.compute-node-version-vars.outputs.NODE_VERSIONS_EXT_STORAGE) }}
     steps:
       - name: Check out code
         uses: actions/checkout@v4
@@ -274,22 +287,35 @@ jobs:
           path: /tmp/docker_cache
           key: docker-images-${{ hashFiles('./images/image-list.txt') }}
 
+      - name: Install mkcert
+        run: curl -JLO "https://dl.filippo.io/mkcert/latest?for=linux/amd64" && sudo chmod 777 mkcert-v*-linux-amd64 && sudo cp mkcert-v*-linux-amd64 /usr/local/bin/mkcert
+
+      - name: Install grep
+        run: sudo apt update && sudo apt install grep
+
+      - name: Check mkcert
+        run: command -v mkcert
+
+      - name: Check grep
+        run: command -v grep
+
       - name: Compile e2e code
         run: yarn build
         working-directory: ./e2e
 
-      - name: Test ${{ matrix.search-version }}
-        run: NODE_VERSION=${{ matrix.node-version }} yarn test:${{ matrix.search-version }}
+      - name: Test external Asset Storage opensearch1
+        run: ENCRYPT_MINIO=true NODE_VERSION=${{ matrix.node-version }} yarn test:s3AssetStorage
         working-directory: ./e2e
 
-  e2e-k8s-tests:
+  teraslice-elasticsearch-tests:
     runs-on: ubuntu-latest
     needs: [compute-node-version-vars, verify-build, cache-docker-images]
     strategy:
       # opensearch is finiky, keep testing others if it fails
       fail-fast: false
       matrix:
-        node-version: ${{ fromJSON(needs.compute-node-version-vars.outputs.NODE_VERSIONS_EXT_K8S) }}
+        node-version: ${{ fromJSON(needs.compute-node-version-vars.outputs.NODE_VERSIONS) }}
+        search-version: [elasticsearch6, elasticsearch7, opensearch1, opensearch2]
     steps:
       - name: Check out code
         uses: actions/checkout@v4
@@ -322,27 +348,19 @@ jobs:
           path: /tmp/docker_cache
           key: docker-images-${{ hashFiles('./images/image-list.txt') }}
 
-      - name: Compile e2e code
-        run: yarn build
-        working-directory: ./e2e
-
-      - name: Install Kind and Kubectl
-        uses: helm/kind-action@v1.10.0
-        with:
-          install_only: "true"
-
-      - name: Test k8s elasticsearch7
-        run: NODE_VERSION=${{ matrix.node-version }} yarn test:k8s
-        working-directory: ./e2e
+      - name: Test ${{ matrix.search-version }}
+        run: yarn test:${{ matrix.search-version }}
+        working-directory: ./packages/teraslice
 
-  e2e-k8s-v2-tests:
+  elasticsearch-store-tests:
     runs-on: ubuntu-latest
     needs: [compute-node-version-vars, verify-build, cache-docker-images]
     strategy:
       # opensearch is finiky, keep testing others if it fails
       fail-fast: false
       matrix:
-        node-version: ${{ fromJSON(needs.compute-node-version-vars.outputs.NODE_VERSIONS_EXT_K8S) }}
+        node-version: ${{ fromJSON(needs.compute-node-version-vars.outputs.NODE_VERSIONS) }}
+        search-version: [elasticsearch6, elasticsearch7, opensearch1, opensearch2]
     steps:
       - name: Check out code
         uses: actions/checkout@v4
@@ -375,27 +393,43 @@ jobs:
           path: /tmp/docker_cache
           key: docker-images-${{ hashFiles('./images/image-list.txt') }}
 
-      - name: Compile e2e code
-        run: yarn build
-        working-directory: ./e2e
+      - name: Test ${{ matrix.search-version }}
+        run: yarn test:${{ matrix.search-version }}
+        working-directory: ./packages/elasticsearch-store
 
-      - name: Install Kind and Kubectl
-        uses: helm/kind-action@v1.10.0
-        with:
-          install_only: "true"
+  lint-and-sync:
+    runs-on: ubuntu-latest
+    needs: [compute-node-version-vars, verify-build, cache-docker-images]
+    # will remove the checkout, build and setup when the artifact is made to just
+    # test the linting and syncing of the codebase
+    steps:
+    - name: Check out code
+      uses: actions/checkout@v4
 
-      - name: Test k8s V2 elasticsearch7
-        run: NODE_VERSION=${{ matrix.node-version }} yarn test:k8sV2
-        working-directory: ./e2e
+    - name: Setup Node
+      uses: actions/setup-node@v4
+      with:
+        node-version: ${{ fromJSON(needs.compute-node-version-vars.outputs.NODE_VERSION_MAIN) }}
+        cache: 'yarn'
 
-  e2e-external-storage-tests:
+    - name: Install and build packages
+      run: yarn && yarn setup
+
+    - name: Lint codebase
+      run: yarn lint
+
+    - name: Sync codebase
+      run: yarn sync --verify
+
+  elasticsearch-api-tests:
     runs-on: ubuntu-latest
     needs: [compute-node-version-vars, verify-build, cache-docker-images]
     strategy:
       # opensearch is finiky, keep testing others if it fails
       fail-fast: false
       matrix:
-        node-version: ${{ fromJSON(needs.compute-node-version-vars.outputs.NODE_VERSIONS_EXT_STORAGE) }}
+        node-version: ${{ fromJSON(needs.compute-node-version-vars.outputs.NODE_VERSIONS) }}
+        search-version: [elasticsearch6, elasticsearch7, opensearch1, opensearch2]
     steps:
       - name: Check out code
         uses: actions/checkout@v4
@@ -428,22 +462,19 @@ jobs:
           path: /tmp/docker_cache
           key: docker-images-${{ hashFiles('./images/image-list.txt') }}
 
-      - name: Compile e2e code
-        run: yarn build
-        working-directory: ./e2e
-
-      - name: Test external Asset Storage opensearch1
-        run: NODE_VERSION=${{ matrix.node-version }} yarn test:s3AssetStorage
-        working-directory: ./e2e
+      - name: Test ${{ matrix.search-version }}
+        run: yarn test:${{ matrix.search-version }}
+        working-directory: ./packages/elasticsearch-api
 
-  e2e-external-storage-tests-encrypted:
+  e2e-tests:
     runs-on: ubuntu-latest
     needs: [compute-node-version-vars, verify-build, cache-docker-images]
     strategy:
       # opensearch is finiky, keep testing others if it fails
       fail-fast: false
       matrix:
-        node-version: ${{ fromJSON(needs.compute-node-version-vars.outputs.NODE_VERSIONS_EXT_STORAGE) }}
+        node-version: ${{ fromJSON(needs.compute-node-version-vars.outputs.NODE_VERSIONS) }}
+        search-version: [elasticsearch6, elasticsearch7, opensearch1, opensearch2]
     steps:
       - name: Check out code
         uses: actions/checkout@v4
@@ -476,24 +507,12 @@ jobs:
           path: /tmp/docker_cache
           key: docker-images-${{ hashFiles('./images/image-list.txt') }}
 
-      - name: Install mkcert
-        run: curl -JLO "https://dl.filippo.io/mkcert/latest?for=linux/amd64" && sudo chmod 777 mkcert-v*-linux-amd64 && sudo cp mkcert-v*-linux-amd64 /usr/local/bin/mkcert
-
-      - name: Install grep
-        run: sudo apt update && sudo apt install grep
-
-      - name: Check mkcert
-        run: command -v mkcert
-
-      - name: Check grep
-        run: command -v grep
-
       - name: Compile e2e code
         run: yarn build
         working-directory: ./e2e
 
-      - name: Test external Asset Storage opensearch1
-        run: ENCRYPT_MINIO=true NODE_VERSION=${{ matrix.node-version }} yarn test:s3AssetStorage
+      - name: Test ${{ matrix.search-version }}
+        run: NODE_VERSION=${{ matrix.node-version }} yarn test:${{ matrix.search-version }}
         working-directory: ./e2e
 
   check-docker-limit-after:

diff --git a/e2e/helm/elasticsearch/README.md b/e2e/helm/elasticsearch/README.md
@@ -0,0 +1,7 @@
+The es binaries in this directory are patched charts from https://artifacthub.io/packages/helm/elastic/elasticsearch (specifically charts versions 7.9.3 and 6.8.9)
+
+Since these charts are no longer maintained, they have incompatible kubernetes api resources that needed to be updated.
+
+The es charts for 6.8.9 and 7.9.3 have a lot files in them and I didn't want to push up 140+ files to the teraslice repo.
+
+In the future we can potentially place these charts in the es assets repo and submit them to artifact hub to get rid of these binaries.
diff --git a/e2e/helm/elasticsearch/elasticsearch-6.8.9.tgz b/e2e/helm/elasticsearch/elasticsearch-6.8.9.tgz
diff --git a/e2e/helm/elasticsearch/elasticsearch-7.9.3.tgz b/e2e/helm/elasticsearch/elasticsearch-7.9.3.tgz