diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml new file mode 100644 index 000000000..8c3d4c6ff --- /dev/null +++ b/.github/workflows/e2e.yaml @@ -0,0 +1,122 @@ +# Copyright 2024 NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: e2e + +on: + pull_request: + types: + - opened + - synchronize + - labeled + branches: + - main + - release-* + push: + branches: + - main + - release-* + +jobs: + build: + runs-on: ubuntu-latest + if: contains(github.event.pull_request.labels.*.name, 'ok-to-test') || github.event_name == 'push' + steps: + - uses: actions/checkout@v4 + name: Check out code + - name: Calculate build vars + id: vars + run: | + echo "COMMIT_SHORT_SHA=${GITHUB_SHA:0:8}" >> $GITHUB_ENV + echo "LOWERCASE_REPO_OWNER=$(echo "${GITHUB_REPOSITORY_OWNER}" | awk '{print tolower($0)}')" >> $GITHUB_ENV + REPO_FULL_NAME="${{ github.event.pull_request.head.repo.full_name }}" + echo "${REPO_FULL_NAME}" + echo "LABEL_IMAGE_SOURCE=https://github.com/${REPO_FULL_NAME}" >> $GITHUB_ENV + + GENERATE_ARTIFACTS="false" + if [[ "${{ github.actor }}" == "dependabot[bot]" ]]; then + GENERATE_ARTIFACTS="false" + fi + echo "PUSH_ON_BUILD=${GENERATE_ARTIFACTS}" >> $GITHUB_ENV + echo "BUILD_MULTI_ARCH_IMAGES=false" >> $GITHUB_ENV + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build image + env: + IMAGE_NAME: ghcr.io/${LOWERCASE_REPO_OWNER}/k8s-device-plugin + VERSION: ${COMMIT_SHORT_SHA} + run: | + echo "${VERSION}" + make -f deployments/container/Makefile build-ubuntu20.04 + + test: + runs-on: ubuntu-latest + needs: build + steps: + - uses: actions/checkout@v4 + name: Checkout code + - name: Calculate test env vars + id: vars + run: | + echo "COMMIT_SHORT_SHA=${GITHUB_SHA:0:8}" >> $GITHUB_ENV + echo "GH_ACTION=gh-action-${GITHUB_SHA:0:8}" >> $GITHUB_ENV + - name: Install Go + uses: actions/setup-go@v5 + with: + go-version: 'stable' + - name: Install YQ + run: | + sudo add-apt-repository ppa:rmescandon/yq -y + sudo apt-get update + sudo apt-get install yq -y + - name: Get private key + run: | + cat << EOF > ./key.pem + ${{ secrets.AWS_SSH_KEY }} + EOF + chmod 0400 ./key.pem + - name: Install and setup Holodeck + env: + HOLODECK_VERSION: v0.1.1 + SSH_KEY: ${{ github.workspace }}/key.pem + run: | + curl -sL https://github.com/NVIDIA/holodeck/releases/download/${HOLODECK_VERSION}/holodeck-linux-amd64 -o /usr/local/bin/holodeck + chmod +x /usr/local/bin/holodeck + yq -i --arg hi "${{ env.GH_ACTION }}" '.metadata.name = "$hi"' holodeck.yaml + yq -i --arg key "${{ env.SSH_KEY }}" '.spec.auth.privateKey = "$key"' holodeck.yaml + - name: Create test environment + env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + run: | + holodeck create -f holodeck.yaml -p -c ${{ github.workspace }}/${COMMIT_SHORT_SHA} -k ./kubeconfig + - name: Run tests + env: + KUBECONFIG: ${{ github.workspace }}/kubeconfig + E2E_IMAGE_REPO: ghcr.io/nvidia/k8s-device-plugin + E2E_IMAGE_TAG: ${COMMIT_SHORT_SHA}-ubuntu20.04 + run: | + echo "Running e2e-tests" + make -f tests/e2e/Makefile e2e-test + - name: Cleanup + run: | + holodeck delete -f holodeck.yaml -c ${{ github.workspace }}/${COMMIT_SHORT_SHA} diff --git a/.github/workflows/golang.yaml b/.github/workflows/golang.yaml index 80066b9c6..4300eb646 100644 --- a/.github/workflows/golang.yaml +++ b/.github/workflows/golang.yaml @@ -12,10 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -name: Golang +name: golang on: pull_request: + types: + - opened + - synchronize + - labeled branches: - main - release-* @@ -27,6 +31,7 @@ on: jobs: check: runs-on: ubuntu-latest + if: contains(github.event.pull_request.labels.*.name, 'ok-to-test') || github.event_name == 'push' steps: - uses: actions/checkout@v4 name: Checkout code @@ -43,6 +48,7 @@ jobs: test: name: Unit test runs-on: ubuntu-latest + if: contains(github.event.pull_request.labels.*.name, 'ok-to-test') || github.event_name == 'push' steps: - name: Checkout code uses: actions/checkout@v4 @@ -53,6 +59,7 @@ jobs: - run: make test build: runs-on: ubuntu-latest + if: contains(github.event.pull_request.labels.*.name, 'ok-to-test') || github.event_name == 'push' steps: - uses: actions/checkout@v4 name: Checkout code diff --git a/.github/workflows/image.yaml b/.github/workflows/image.yaml index 7c2fc75df..bd2281985 100644 --- a/.github/workflows/image.yaml +++ b/.github/workflows/image.yaml @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Run this workflow on pull requests or merge to main/release branches name: image on: @@ -20,6 +19,7 @@ on: types: - opened - synchronize + - labeled branches: - main - release-* @@ -31,6 +31,7 @@ on: jobs: build: runs-on: ubuntu-latest + if: contains(github.event.pull_request.labels.*.name, 'ok-to-test') || github.event_name == 'push' strategy: matrix: dist: [ubuntu20.04, ubi8] diff --git a/holodeck b/holodeck new file mode 100755 index 000000000..d8d1d9f0b Binary files /dev/null and b/holodeck differ diff --git a/kubeconfig b/kubeconfig new file mode 100644 index 000000000..33cf6ba55 --- /dev/null +++ b/kubeconfig @@ -0,0 +1,19 @@ +apiVersion: v1 +clusters: +- cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURCVENDQWUyZ0F3SUJBZ0lJT3hwN1VjZlNjc3N3RFFZSktvWklodmNOQVFFTEJRQXdGVEVUTUJFR0ExVUUKQXhNS2EzVmlaWEp1WlhSbGN6QWVGdzB5TkRBeU1qRXhNRFUwTXpWYUZ3MHpOREF5TVRneE1EVTVNelZhTUJVeApFekFSQmdOVkJBTVRDbXQxWW1WeWJtVjBaWE13Z2dFaU1BMEdDU3FHU0liM0RRRUJBUVVBQTRJQkR3QXdnZ0VLCkFvSUJBUUN5c1VmWm40YW0ySVRLS3FOTG5tWXpjSWY0RFhqODJVT2N2a0dmUDR1UjYxOWlZYStLdGpYM3lMQkIKY0oySGVRZjNQM3dDaTNOSE1wOEpvanU4dEFpd2FFdEJXZWlHUUxtZTA0QXN6aVg2emluSS8zWHVoN1pTcDdRTQpxdHJwREFabytyRnhieVljZjVNL2YvVHNhQ3RIOUVPUks5dDU4MEFVSXJkMTE5bXJ1MkxDNXd2UU9ZUUhveXdDCnVCRFhnaVVDVDMydEJFYnF4T1FrSXNLbTYySXVtZDJzRm4zTHBlMjBENGRyY2dpOWM2eWovOVJnM2w3S2RMWm4KaEN3bDZteHViRFExOHBnVWFCeEdiTGVYSzlGcWxZcE9MQWJRckhnYWE3c0hEd1VnRmRMSDRvdksySVJXc1BySApCeUFBSTMyRHJwVzZVVEtqVjJ0Z21sSUErMlpCQWdNQkFBR2pXVEJYTUE0R0ExVWREd0VCL3dRRUF3SUNwREFQCkJnTlZIUk1CQWY4RUJUQURBUUgvTUIwR0ExVWREZ1FXQkJRNVhHc1pWK1BCZXBQUWhBcFBiV1lvNWpOUGVEQVYKQmdOVkhSRUVEakFNZ2dwcmRXSmxjbTVsZEdWek1BMEdDU3FHU0liM0RRRUJDd1VBQTRJQkFRQmFrWGV1NENhdwo5TDNZdnF6clZqSFpZempjNGJFRWN5ZkdHRHUvNUtlWDF6WFNJWXRtZnhsc0lOc3VPQWgwdHA2ZUl5NnBEcktBCmMyTnVqUjcrS1NmTEFCVmZEamV5S0tKZGhIbzZiR0xSWnp2SzIrNnFmQThrU1FyUEx6TmxVdThMd3p1RjZjQWQKWjl5bWh6amhQeGdaeFNOaEN2eTExRFN1Y1kwU21PWEZRak1hRllsZXFJK3oxay9mRFBUbWJIQk9nTE8zMXZ5YQpOckFKNmNmSThRdU9SZVRmcnpmaVlXM0l0dzFMUDhFT1AwK3d4cGNraTdhYnFQMDNBM3BKejUwRTQ0Sk5zaHFhCldlTDFWQUJhVXdDQUtyVG5wZHpxOHdXMlBqdjY3K014eE5SSGRPYkIrNERMZXF4RTYzaWVybG5kMktGbm5WOXAKTnNMb0pjc3FyeW9VCi0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K + server: https://ec2-54-153-26-145.us-west-1.compute.amazonaws.com:6443 + name: kubernetes +contexts: +- context: + cluster: kubernetes + user: kubernetes-admin + name: kubernetes-admin@kubernetes +current-context: kubernetes-admin@kubernetes +kind: Config +preferences: {} +users: +- name: kubernetes-admin + user: + client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURJVENDQWdtZ0F3SUJBZ0lJWm1td2UrbkpZWmd3RFFZSktvWklodmNOQVFFTEJRQXdGVEVUTUJFR0ExVUUKQXhNS2EzVmlaWEp1WlhSbGN6QWVGdzB5TkRBeU1qRXhNRFUwTXpWYUZ3MHlOVEF5TWpBeE1EVTVNemRhTURReApGekFWQmdOVkJBb1REbk41YzNSbGJUcHRZWE4wWlhKek1Sa3dGd1lEVlFRREV4QnJkV0psY201bGRHVnpMV0ZrCmJXbHVNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DQVE4QU1JSUJDZ0tDQVFFQXVMMzFhSHZlUE43b2c4UC8KMjRZb1dKQzlYN2hrbFdZQVc3QjdFcUhLeWpLOG1HamdxOVFSTG1pMDN0YkoxTmphNVhBWWdJSzNiZ21zYS9XTQorYnRCVWNTdWxoM21rMmVyZEd6ZkV1WWlVN2pFcDROSWk5OHFIMVptMFl0NUNJM0lVWDNvOWNjaWRNNWI2WU01ClhxL1NUakZqWjhSdExKa1hYbXkvZ2N2VUNYczZRQjMrSWdFaG1vK093OVFIV3dkNGlCZG9WWFlJVURxT2U5bDUKeDMrUWJwZngzU04ySVBkVndrSVdCL3ZERG1CcWpRN01iaGF6Q0xQRis5SWdjOGt6b25YOXNaTTlkcGlEVm14NQpmdDYwVFhJSkZkTEZwUERzRk9XVEJVaDZFdUJIY2RTdm9qV29ObGxzOW02UHhPSExheGZuRWtGc0lOa1l0TFZUCkdDTDRsUUlEQVFBQm8xWXdWREFPQmdOVkhROEJBZjhFQkFNQ0JhQXdFd1lEVlIwbEJBd3dDZ1lJS3dZQkJRVUgKQXdJd0RBWURWUjBUQVFIL0JBSXdBREFmQmdOVkhTTUVHREFXZ0JRNVhHc1pWK1BCZXBQUWhBcFBiV1lvNWpOUAplREFOQmdrcWhraUc5dzBCQVFzRkFBT0NBUUVBcm5MenlsSE5vSE5waFVMRm8vT1hlbkxVS0lqb0Q5a2xOTUQrClA2MWE0b1o5aFFqN09UU1lIbmJUR0NlV3FiVis5S0VPZC92aE1iSmZvWmkvRkdMV25kQ21kM1I2K2hFZkZDZUUKbzNITHl2Z1JENVhKTE1EWGhkV0dmcENUcEs1WVp2dms2QmtWeXBkRTZEaSswVjZGVGNyRnFaMUJwQzdKRTJ6RwpkZTJmN0E0TVRQUjZSWndWM3NmeWFsSEpNMHF6RWNUUUE4bXdGcXk3SittOCt1bm8wVHhtSzZuSU8xeFg1NDlECkhzL0tQc1V4VDZQam84MEt2S2crSEtvVEphRGNTV1FUdjJyVHg2c2tzOEJ5QUs3RWZ0cXFYdnFqMysyOVljbkMKaGIrS2w0VUxLZUhIMVZGeGVVMWprQlZtdUlsYklPbFN3TUd0Yjh4L3I2Zm4reEg2Snc9PQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg== + client-key-data: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb3dJQkFBS0NBUUVBdUwzMWFIdmVQTjdvZzhQLzI0WW9XSkM5WDdoa2xXWUFXN0I3RXFIS3lqSzhtR2pnCnE5UVJMbWkwM3RiSjFOamE1WEFZZ0lLM2JnbXNhL1dNK2J0QlVjU3VsaDNtazJlcmRHemZFdVlpVTdqRXA0TkkKaTk4cUgxWm0wWXQ1Q0kzSVVYM285Y2NpZE01YjZZTTVYcS9TVGpGalo4UnRMSmtYWG15L2djdlVDWHM2UUIzKwpJZ0VobW8rT3c5UUhXd2Q0aUJkb1ZYWUlVRHFPZTlsNXgzK1FicGZ4M1NOMklQZFZ3a0lXQi92RERtQnFqUTdNCmJoYXpDTFBGKzlJZ2M4a3pvblg5c1pNOWRwaURWbXg1ZnQ2MFRYSUpGZExGcFBEc0ZPV1RCVWg2RXVCSGNkU3YKb2pXb05sbHM5bTZQeE9ITGF4Zm5Fa0ZzSU5rWXRMVlRHQ0w0bFFJREFRQUJBb0lCQURwdy9UMDhVaVYzYnA2ZwpWa1Fmd0l1ajNnUjExSHZlZEthTXdhSTNSR3VLZVZiTmRxVmt4THFsclBJQmlIK295alppcTUyYnZHTTI5RENDCjBVSC9GcjdETldlUHRlMVRrazJXVTBzck9oaTZEcHZmVUFmUFpRZmMwVkhnaVQxSGZPTXFyWk55bDFmVUlBZFMKMThjNFg1TVUxSmx5N2daNXQ0YkxhQm1yWFR4U0VYNTQvdE1NZGJFaEI1cE54ZGVLMms5OWovYUlwcm5FMGVkOQp6d1RFcEE2OHJHZWl1RTgzMVdnTEI4TGMwUllLc3UrVWFqZUVUNVdkMXVCUC96WFJLVTY4VVZSekpHOWdqL2plCnZWZDNsaE14NzJxYnU1R1NGZDYwMzRDMFJHOHljYTY4QktscDliZ0lnWXNaRXltalBHVnJIekNkNEEzdDFndUsKKzFQUEVWRUNnWUVBNGR0UEtkdXlVQ3VQanJ6NDlFRnFPTXpaWnFJUDI0Ti9sY1lWRlBzS3RyRXQ2ejl0WjNzMQptSU5vaGlxd0FxSHBzR29vKytIWmh3U3grbW9wRUZvUFhOMWx3RldUakNGbmprNUt5N0NHcFBZL2UvWlBPV0ZhCllMU2hQc2V0UHNKNDJVU01MeVJ2dXA4REhraldsVmtXQ1NzdGhpekdObzdTbjlaNUdTYTc3UXNDZ1lFQTBXWHAKeUFaYWtIYVNvdnJpeHlyOTR2NUU0UUIrS2dEK01zcWRQbnpqeG53UEo1c3o5Vzc2b2FNRERjcVJzUk9vdjlaeQpVRHBFR1JNYWZEZnR1VmhtYlI3OGxLemx3RGZYQlp3Z2FyUkE0RHhIVUNxV2xpQloyU2kzVmhrYnZIczlydDI5Cm5rNTQ3N0MvSmNsRk4rR0VkZTQ2Q21ud0tXd2VPdW5IeGZ3ajlOOENnWUI2ZUNKQUQxcHNjSUtDb3VQWUJ0U2oKZ2lqc3B6bzBET1VWQVBrVGpHRXQyWHVOMGJId1ovWVRBa2FlWWUxWk1CVDR6VWYxOVd5djh4TFROOWVXZjA5UApTdHBPNWRpQkp1ZnhXWERqYkpCR1NTVDVnZ0ZKbXdtWDRMcUN3V0dtNDJzR1hVNHFqZlpyMnFlbE4xZnNXZ2RCCjU0RGkvY3BwY1I2eS8yaGhCQWpwS3dLQmdRREJ5dzBTeHZhWURuVGxpMFhJU0R5U0VQclVOZTJObUQrcDZDU2gKQ1ZxUWZvNVQvR3FZOGUzWkhCR2VlT1JmT3ErSlk4R0dDSGF2c2lObjVTY0l5d3g0WHg0cHh0NFhDRmJpNVR5SApKRzJLV1o4eVRVZnlodHhBZzFNeHhxa0p4aEV2UGg3WVhRM0YzeEQ0dW81cUNqZmFJUERzWkdFVkpPL1JYRDA4Ck9YOVpod0tCZ0EyMWhHY003d0h1YzJybDF0eTNZTWF2SU5MaVdKOEpjV3lFV25YcmxmbXdKUTZtYjVtQzdpMnkKM1hiQ3ZYRitIaTZLWmFWb1lMdHpEZVZvcWt1ZGlQdXlRcklnY2tUQnkzUnVZenRnWHo0U3BwVUc2Wld4WmtIUApYUVl5ZG1mWllGSEVEMFRxRkJTRGV0dmlCbU5Xd2xLYm15V2dleFdSS1ZnM3dTMXVFSy9FCi0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg== diff --git a/tests/e2e/Makefile b/tests/e2e/Makefile index 8602a886f..7ed4875d9 100644 --- a/tests/e2e/Makefile +++ b/tests/e2e/Makefile @@ -15,13 +15,14 @@ GO_CMD ?= go GO_FMT ?= gofmt -DRIVER_ENABLED ?= true +include $(CURDIR)/versions.mk -E2E_IMAGE_REPO ?= nvcr.io/nvidia/k8s-device-plugin -E2E_IMAGE_TAG ?= v0.14.3 -E2E_IMAGE_PULL_POLICY ?= Always +DRIVER_ENABLED ?= true -HELM_CHART ?= $(CURDIR)/../../deployments/helm/nvidia-device-plugin +E2E_IMAGE_REPO ?= $(REGISTRY)/$(DRIVER_NAME) +E2E_IMAGE_TAG ?= $(VERSION) +E2E_IMAGE_PULL_POLICY ?= IfNotPresent +HELM_CHART ?= $(CURDIR)/deployments/helm/nvidia-device-plugin .PHONY: e2e-test e2e-test: @@ -29,7 +30,7 @@ e2e-test: echo "[ERR] KUBECONFIG missing, must be defined"; \ exit 1; \ fi - $(GO_CMD) test -v . -args \ + $(GO_CMD) test -v $(CURDIR)/tests/e2e -args \ -kubeconfig=$(KUBECONFIG) \ -driver-enabled=$(DRIVER_ENABLED) \ -image.repo=$(E2E_IMAGE_REPO) \ diff --git a/tests/e2e/holodeck.yaml b/tests/e2e/holodeck.yaml new file mode 100644 index 000000000..3a5a77ce0 --- /dev/null +++ b/tests/e2e/holodeck.yaml @@ -0,0 +1,29 @@ +apiVersion: holodeck.nvidia.com/v1alpha1 +kind: Environment +metadata: + name: gh-action-5b6033b2 + description: "Devel infra environment" +spec: + provider: aws + auth: + keyName: cnt-ci + privateKey: "/Users/eduardoa/.ssh/cnt-ci.pem" + instance: + type: g4dn.xlarge + region: us-west-1 + ingressIpRanges: + - 0.0.0.0/0 + image: + architecture: amd64 + imageId: ami-0ce2cb35386fc22e9 + containerRuntime: + install: true + name: containerd + nvidiaContainerToolkit: + install: true + nvidiaDriver: + install: true + kubernetes: + install: true + installer: kubeadm + version: v1.28.5