From e010068bb5f77cf940e4312ad26872a5cc646f33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20M=C3=A1gr?= Date: Tue, 10 Oct 2023 11:46:27 +0200 Subject: [PATCH 1/3] Add kuttl tests for autoscaling --- Makefile | 16 ++++++++++++++++ hack/install-krew.sh | 15 +++++++++++++++ 2 files changed, 31 insertions(+) create mode 100755 hack/install-krew.sh diff --git a/Makefile b/Makefile index f291540c..ae9630fb 100644 --- a/Makefile +++ b/Makefile @@ -325,3 +325,19 @@ run-with-webhook: export HEALTH_PORT?=8081 run-with-webhook: manifests generate fmt vet ## Run a controller from your host. /bin/bash hack/configure_local_webhook.sh go run ./main.go -metrics-bind-address ":$(METRICS_PORT)" -health-probe-bind-address ":$(HEALTH_PORT)" + +.PHONY: kuttl-install +kuttl-install: + ./hack/install-krew.sh + export PATH="$${KREW_ROOT:-$$HOME/.krew}/bin:$$PATH" && kubectl krew install kuttl + echo "Place 'export PATH=$${KREW_ROOT:-$$HOME/.krew}/bin:$$PATH' to your ~/.bashrc" + +# TODO(mmagr): change this to default once there is default suite +KUTTL_SUITE ?= autoscaling +KUTTL_NAMESPACE ?= telemetry-kuttl-$(KUTTL_SUITE) +KUTTL_SUITE_DIR ?= tests/kuttl/suites/$(KUTTL_SUITE) + +.PHONY: kuttl-test-prep +kuttl-test-prep: + oc apply -k tests/kuttl/deps/ --timeout=120s + oc wait -n $(KUTTL_NAMESPACE) openstackcontrolplane openstack --for condition=Ready --timeout=300s diff --git a/hack/install-krew.sh b/hack/install-krew.sh new file mode 100755 index 00000000..03c066c2 --- /dev/null +++ b/hack/install-krew.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +# taken from https://github.com/openstack-k8s-operators/nova-operator/blob/main/hack/install-krew.sh +( + set -x + tdir="$(mktemp -d)" + pushd $tdir && + OS="$(uname | tr '[:upper:]' '[:lower:]')" && + ARCH="$(uname -m | sed -e 's/x86_64/amd64/' -e 's/\(arm\)\(64\)\?.*/\1\2/' -e 's/aarch64$/arm64/')" && + KREW="krew-${OS}_${ARCH}" && + curl -fsSLO "https://github.com/kubernetes-sigs/krew/releases/latest/download/${KREW}.tar.gz" && + tar zxvf "${KREW}.tar.gz" && + ./"${KREW}" install krew + popd && rm -fr $tdir +) From b8384213ab66c2c77e0781a6bdc26c10ab801def Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20M=C3=A1gr?= Date: Tue, 10 Oct 2023 13:46:30 +0200 Subject: [PATCH 2/3] kuttl deps deploy --- Makefile | 4 +- .../deps/OpenStackControlPlane.yaml | 35 ++++++++++++++ .../kuttl/suites/autoscaling/deps/infra.yaml | 41 +++++++++++++++++ .../autoscaling/deps/kustomization.yaml | 46 +++++++++++++++++++ .../suites/autoscaling/deps/namespace.yaml | 4 ++ .../kuttl/suites/autoscaling/deps/rhobs.yaml | 33 +++++++++++++ .../suites/autoscaling/deps/telemetry.yaml | 7 +++ 7 files changed, 169 insertions(+), 1 deletion(-) create mode 100644 tests/kuttl/suites/autoscaling/deps/OpenStackControlPlane.yaml create mode 100644 tests/kuttl/suites/autoscaling/deps/infra.yaml create mode 100644 tests/kuttl/suites/autoscaling/deps/kustomization.yaml create mode 100644 tests/kuttl/suites/autoscaling/deps/namespace.yaml create mode 100644 tests/kuttl/suites/autoscaling/deps/rhobs.yaml create mode 100644 tests/kuttl/suites/autoscaling/deps/telemetry.yaml diff --git a/Makefile b/Makefile index ae9630fb..140f520e 100644 --- a/Makefile +++ b/Makefile @@ -339,5 +339,7 @@ KUTTL_SUITE_DIR ?= tests/kuttl/suites/$(KUTTL_SUITE) .PHONY: kuttl-test-prep kuttl-test-prep: - oc apply -k tests/kuttl/deps/ --timeout=120s + oc apply -k $(KUTTL_SUITE_DIR)/deps/ --timeout=120s oc wait -n $(KUTTL_NAMESPACE) openstackcontrolplane openstack --for condition=Ready --timeout=300s + oc apply -f $(KUTTL_SUITE_DIR)/deps/rhobs.yaml + until oc api-resources | grep -q rhobs; do sleep 2; done diff --git a/tests/kuttl/suites/autoscaling/deps/OpenStackControlPlane.yaml b/tests/kuttl/suites/autoscaling/deps/OpenStackControlPlane.yaml new file mode 100644 index 00000000..6210b48b --- /dev/null +++ b/tests/kuttl/suites/autoscaling/deps/OpenStackControlPlane.yaml @@ -0,0 +1,35 @@ +apiVersion: core.openstack.org/v1beta1 +kind: OpenStackControlPlane +metadata: + name: openstack +spec: + storageClass: "crc-csi-hostpath-provisioner" + keystone: + template: + databaseInstance: openstack + secret: osp-secret + ironic: + enabled: false + template: + ironicConductors: [] + manila: + enabled: false + template: + manilaShares: {} + horizon: + enabled: false + nova: + enabled: false + placement: + template: + databaseInstance: openstack + secret: osp-secret + heat: + enabled: true + template: + databaseInstance: openstack + heatAPI: + replicas: 1 + heatEngine: + replicas: 1 + secret: osp-secret diff --git a/tests/kuttl/suites/autoscaling/deps/infra.yaml b/tests/kuttl/suites/autoscaling/deps/infra.yaml new file mode 100644 index 00000000..96d2d55d --- /dev/null +++ b/tests/kuttl/suites/autoscaling/deps/infra.yaml @@ -0,0 +1,41 @@ +apiVersion: core.openstack.org/v1beta1 +kind: OpenStackControlPlane +metadata: + name: openstack +spec: + mariadb: + enabled: false + templates: + openstack: + replicas: 0 + openstack-cell1: + replicas: 0 + galera: + enabled: true + templates: + openstack: + replicas: 1 + storageRequest: 500M + openstack-cell1: + replicas: 1 + storageRequest: 500M + secret: osp-secret + secret: osp-secret + rabbitmq: + templates: + rabbitmq: + replicas: 1 + rabbitmq-cell1: + replicas: 1 + memcached: + templates: + memcached: + replicas: 1 + ovn: + enabled: false + template: + ovnController: + external-ids: + ovn-encap-type: geneve + ovs: + enabled: false diff --git a/tests/kuttl/suites/autoscaling/deps/kustomization.yaml b/tests/kuttl/suites/autoscaling/deps/kustomization.yaml new file mode 100644 index 00000000..3703d065 --- /dev/null +++ b/tests/kuttl/suites/autoscaling/deps/kustomization.yaml @@ -0,0 +1,46 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: telemetry-kuttl-autoscaling + +secretGenerator: +- literals: + - AdminPassword=password + - DbRootPassword=password + - DatabasePassword=password + - KeystoneDatabasePassword=password + - PlacementPassword=password + - PlacementDatabasePassword=password + - GlancePassword=password + - GlanceDatabasePassword=password + - NeutronPassword=password + - NeutronDatabasePassword=password + - NovaPassword=password + - NovaAPIDatabasePassword=password + - NovaCell0DatabasePassword=password + - NovaCell1DatabasePassword=password + - AodhPassword=password + - CeilometerPassword=password + - HeatPassword=password + - HeatDatabasePassword=password + - HeatAuthEncryptionKey=66699966699966600666999666999666 + - MetadataSecret=42 + name: osp-secret +generatorOptions: + disableNameSuffixHash: true + labels: + type: osp-secret + +resources: +- namespace.yaml +- OpenStackControlPlane.yaml + +patches: +- patch: |- + apiVersion: core.openstack.org/v1beta1 + kind: OpenStackControlPlane + metadata: + name: openstack + spec: + secret: osp-secret +- path: infra.yaml +- path: telemetry.yaml diff --git a/tests/kuttl/suites/autoscaling/deps/namespace.yaml b/tests/kuttl/suites/autoscaling/deps/namespace.yaml new file mode 100644 index 00000000..63c85762 --- /dev/null +++ b/tests/kuttl/suites/autoscaling/deps/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: telemetry-kuttl diff --git a/tests/kuttl/suites/autoscaling/deps/rhobs.yaml b/tests/kuttl/suites/autoscaling/deps/rhobs.yaml new file mode 100644 index 00000000..c5ed2a14 --- /dev/null +++ b/tests/kuttl/suites/autoscaling/deps/rhobs.yaml @@ -0,0 +1,33 @@ +apiVersion: operators.coreos.com/v1alpha1 +kind: CatalogSource +metadata: + annotations: + name: observability-operator + namespace: openshift-marketplace +spec: + displayName: Observability Operator - Test + icon: + base64data: "" + mediatype: "" + image: quay.io/rhobs/observability-operator-catalog:latest + publisher: Sunil Thaha + sourceType: grpc + grpcPodConfig: + securityContextConfig: restricted + updateStrategy: + registryPoll: + interval: 1m0s +--- +apiVersion: operators.coreos.com/v1alpha1 +kind: Subscription +metadata: + labels: + operators.coreos.com/observability-operator.openshift-operators: "" + name: observability-operator + namespace: openshift-operators +spec: + channel: development + installPlanApproval: Automatic + name: observability-operator + source: observability-operator + sourceNamespace: openshift-marketplace diff --git a/tests/kuttl/suites/autoscaling/deps/telemetry.yaml b/tests/kuttl/suites/autoscaling/deps/telemetry.yaml new file mode 100644 index 00000000..5c2714a2 --- /dev/null +++ b/tests/kuttl/suites/autoscaling/deps/telemetry.yaml @@ -0,0 +1,7 @@ +apiVersion: core.openstack.org/v1beta1 +kind: OpenStackControlPlane +metadata: + name: openstack +spec: + telemetry: + enabled: false From 1be3b66a6cf25988a8f73ee2ccc56af9907e5d16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20M=C3=A1gr?= Date: Wed, 11 Oct 2023 19:06:14 +0200 Subject: [PATCH 3/3] Autoscaling deploy test --- Makefile | 26 ++++- tests/kuttl/suites/autoscaling/config.yaml | 14 +++ .../autoscaling/deps/kustomization.yaml | 2 + tests/kuttl/suites/autoscaling/output/.keep | 0 .../suites/autoscaling/tests/00-assert.yaml | 101 ++++++++++++++++++ .../suites/autoscaling/tests/00-deploy.yaml | 19 ++++ 6 files changed, 161 insertions(+), 1 deletion(-) create mode 100644 tests/kuttl/suites/autoscaling/config.yaml create mode 100644 tests/kuttl/suites/autoscaling/output/.keep create mode 100644 tests/kuttl/suites/autoscaling/tests/00-assert.yaml create mode 100644 tests/kuttl/suites/autoscaling/tests/00-deploy.yaml diff --git a/Makefile b/Makefile index 140f520e..e28c2188 100644 --- a/Makefile +++ b/Makefile @@ -342,4 +342,28 @@ kuttl-test-prep: oc apply -k $(KUTTL_SUITE_DIR)/deps/ --timeout=120s oc wait -n $(KUTTL_NAMESPACE) openstackcontrolplane openstack --for condition=Ready --timeout=300s oc apply -f $(KUTTL_SUITE_DIR)/deps/rhobs.yaml - until oc api-resources | grep -q rhobs; do sleep 2; done + $(until oc api-resources -n $(KUTTL_NAMESPACE) | grep -q rhobs; do sleep 2; done) + +.PHONY: kuttl-test-run +kuttl-test-run: + oc kuttl test --v 1 --start-kind=false --config $(KUTTL_SUITE_DIR)/config.yaml + +.PHONY: kuttl-test +kuttl-test: kuttl-test-prep kuttl-test-run + +.PHONY: kuttl-test-cleanup +kuttl-test-cleanup: + # only cleanup if the $(KUTTL_NAMESPACE) exists + $(eval namespace_exists=$(shell oc get namespace $(KUTTL_NAMESPACE) --ignore-not-found -o name)) + # We need to order the deletion. Simply deleting the namespace will + # result in errors in mariadb- and keystone-operator and then + # finalizer removal get stuck blocking the namespace deletion. + if [ "${namespace_exists}" != "" ]; then \ + if [ "$(KUTTL_SUITE)" == "autoscaling" ]; then \ + oc delete --wait=true --all=true -n $(KUTTL_NAMESPACE) --timeout=120s Autoscaling; \ + fi; \ + oc delete --wait=true --all=true -n $(KUTTL_NAMESPACE) --timeout=120s OpenStackControlPlane; \ + oc delete --wait=true namespace $(KUTTL_NAMESPACE); \ + else \ + echo "Namespce already cleaned up. Nothing to do"; \ + fi diff --git a/tests/kuttl/suites/autoscaling/config.yaml b/tests/kuttl/suites/autoscaling/config.yaml new file mode 100644 index 00000000..15918e0b --- /dev/null +++ b/tests/kuttl/suites/autoscaling/config.yaml @@ -0,0 +1,14 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestSuite +reportFormat: JSON +reportName: kuttl-autoscaling-results +namespace: telemetry-kuttl-autoscaling +# we could set this lower, but the initial image pull can take a while +timeout: 300 +parallel: 1 +skipDelete: true +testDirs: + - tests/kuttl/suites/autoscaling/ +suppress: + - events +artifactsDir: tests/kuttl/suites/autoscaling/output diff --git a/tests/kuttl/suites/autoscaling/deps/kustomization.yaml b/tests/kuttl/suites/autoscaling/deps/kustomization.yaml index 3703d065..7a480f7c 100644 --- a/tests/kuttl/suites/autoscaling/deps/kustomization.yaml +++ b/tests/kuttl/suites/autoscaling/deps/kustomization.yaml @@ -19,7 +19,9 @@ secretGenerator: - NovaCell0DatabasePassword=password - NovaCell1DatabasePassword=password - AodhPassword=password + - AodhDatabasePassword=password - CeilometerPassword=password + - CeilometerDatabasePassword=password - HeatPassword=password - HeatDatabasePassword=password - HeatAuthEncryptionKey=66699966699966600666999666999666 diff --git a/tests/kuttl/suites/autoscaling/output/.keep b/tests/kuttl/suites/autoscaling/output/.keep new file mode 100644 index 00000000..e69de29b diff --git a/tests/kuttl/suites/autoscaling/tests/00-assert.yaml b/tests/kuttl/suites/autoscaling/tests/00-assert.yaml new file mode 100644 index 00000000..c5fe5342 --- /dev/null +++ b/tests/kuttl/suites/autoscaling/tests/00-assert.yaml @@ -0,0 +1,101 @@ +apiVersion: v1 +kind: Pod +metadata: + labels: + prometheus: telemetry-kuttl-prometheus + name: prometheus-telemetry-kuttl-prometheus-0 +status: + containerStatuses: + - image: quay.io/rhobs/obo-prometheus-config-reloader:v0.68.0-rhobs1 + name: config-reloader + ready: true + started: true + - image: quay.io/prometheus/prometheus:v2.47.0 + name: prometheus + ready: true + started: true + - image: quay.io/thanos/thanos:v0.24.0 + name: thanos-sidecar + ready: true + started: true +--- +apiVersion: v1 +kind: Service +metadata: + name: telemetry-kuttl-prometheus-prometheus + namespace: telemetry-kuttl-autoscaling + ownerReferences: + - kind: MonitoringStack + name: telemetry-kuttl-prometheus +spec: + ports: + - name: web + port: 9090 + protocol: TCP + targetPort: 9090 +--- +apiVersion: v1 +kind: Service +metadata: + labels: + service: aodh + endpoint: internal + name: aodh-internal + namespace: telemetry-kuttl-autoscaling + ownerReferences: + - kind: Autoscaling + name: telemetry-kuttl +spec: + ports: + - name: aodh-internal + port: 8042 + protocol: TCP + targetPort: 8042 +--- +apiVersion: v1 +kind: Service +metadata: + labels: + service: aodh + endpoint: public + name: aodh-public + namespace: telemetry-kuttl-autoscaling + ownerReferences: + - kind: Autoscaling + name: telemetry-kuttl +spec: + ports: + - name: aodh-public + port: 8042 + protocol: TCP + targetPort: 8042 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + service: aodh + name: aodh + namespace: telemetry-kuttl-autoscaling + ownerReferences: + - kind: Autoscaling + name: telemetry-kuttl +spec: + template: + spec: + containers: + - image: quay.io/mmagr/openstack-aodh-api:current-podified + name: aodh-api + - image: quay.io/mmagr/openstack-aodh-evaluator:current-podified + name: aodh-evaluator + - image: quay.io/mmagr/openstack-aodh-notifier:current-podified + name: aodh-notifier + - image: quay.io/mmagr/openstack-aodh-listener:current-podified + name: aodh-listener + volumes: + - name: scripts + secret: + secretName: aodh-scripts + - name: config-data + secret: + secretName: aodh-config-data diff --git a/tests/kuttl/suites/autoscaling/tests/00-deploy.yaml b/tests/kuttl/suites/autoscaling/tests/00-deploy.yaml new file mode 100644 index 00000000..662f8797 --- /dev/null +++ b/tests/kuttl/suites/autoscaling/tests/00-deploy.yaml @@ -0,0 +1,19 @@ +apiVersion: telemetry.openstack.org/v1beta1 +kind: Autoscaling +metadata: + name: telemetry-kuttl +spec: + enabled: true + prometheus: + deployPrometheus: true + aodh: + secret: osp-secret + apiImage: "quay.io/mmagr/openstack-aodh-api:current-podified" + evaluatorImage: "quay.io/mmagr/openstack-aodh-evaluator:current-podified" + notifierImage: "quay.io/mmagr/openstack-aodh-notifier:current-podified" + listenerImage: "quay.io/mmagr/openstack-aodh-listener:current-podified" + passwordSelectors: + databaseUser: aodh + databaseInstance: openstack + memcachedInstance: memcached + heatInstance: heat