From 8219ffcef27b3b8d3b7766469d4ab8e491189503 Mon Sep 17 00:00:00 2001 From: Mike Metral <1112768+metral@users.noreply.github.com> Date: Thu, 18 Jun 2020 09:49:34 -0700 Subject: [PATCH] Fix prometheus-operator test to wait for the CRD to be ready before use --- CHANGELOG.md | 4 + tests/examples/examples_test.go | 57 +++++----- tests/examples/prometheus-operator/index.ts | 100 +++++++++++------ .../examples/prometheus-operator/package.json | 2 + .../prometheus-operator/step1/index.ts | 102 ++++++++++++------ 5 files changed, 171 insertions(+), 94 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 390a915f7a..ff79360712 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ ## HEAD (Unreleased) +### Improvements + +- Fix prometheus-operator test to wait for the CRD to be ready before use (https://github.com/pulumi/pulumi-kubernetes/pull/1172) + ## 2.3.1 (June 17, 2020) ### Improvements diff --git a/tests/examples/examples_test.go b/tests/examples/examples_test.go index 2b6bfd63a5..3d92861cbd 100644 --- a/tests/examples/examples_test.go +++ b/tests/examples/examples_test.go @@ -188,35 +188,34 @@ func TestAccHelmLocal(t *testing.T) { integration.ProgramTest(t, &test) } -// TODO: uncomment once https://github.com/pulumi/pulumi-kubernetes/issues/1137 is fixed. -//func TestAccPrometheusOperator(t *testing.T) { -// skipIfShort(t) -// test := getBaseOptions(t). -// With(integration.ProgramTestOptions{ -// Dir: path.Join(getCwd(t), "prometheus-operator"), -// SkipRefresh: true, -// ExtraRuntimeValidation: func( -// t *testing.T, stackInfo integration.RuntimeValidationStackInfo, -// ) { -// assert.NotNil(t, stackInfo.Deployment) -// assert.Equal(t, 10, len(stackInfo.Deployment.Resources)) -// }, -// EditDirs: []integration.EditDir{ -// { -// Dir: path.Join(getCwd(t), "prometheus-operator", "steps"), -// Additive: true, -// ExtraRuntimeValidation: func( -// t *testing.T, stackInfo integration.RuntimeValidationStackInfo, -// ) { -// assert.NotNil(t, stackInfo.Deployment) -// assert.Equal(t, 10, len(stackInfo.Deployment.Resources)) -// }, -// }, -// }, -// }) -// -// integration.ProgramTest(t, &test) -//} +func TestAccPrometheusOperator(t *testing.T) { + skipIfShort(t) + test := getBaseOptions(t). + With(integration.ProgramTestOptions{ + Dir: path.Join(getCwd(t), "prometheus-operator"), + SkipRefresh: true, + ExtraRuntimeValidation: func( + t *testing.T, stackInfo integration.RuntimeValidationStackInfo, + ) { + assert.NotNil(t, stackInfo.Deployment) + assert.Equal(t, 10, len(stackInfo.Deployment.Resources)) + }, + EditDirs: []integration.EditDir{ + { + Dir: path.Join(getCwd(t), "prometheus-operator", "steps"), + Additive: true, + ExtraRuntimeValidation: func( + t *testing.T, stackInfo integration.RuntimeValidationStackInfo, + ) { + assert.NotNil(t, stackInfo.Deployment) + assert.Equal(t, 10, len(stackInfo.Deployment.Resources)) + }, + }, + }, + }) + + integration.ProgramTest(t, &test) +} func TestAccMariadb(t *testing.T) { skipIfShort(t) diff --git a/tests/examples/prometheus-operator/index.ts b/tests/examples/prometheus-operator/index.ts index 52f1a161eb..eecacdd8f3 100644 --- a/tests/examples/prometheus-operator/index.ts +++ b/tests/examples/prometheus-operator/index.ts @@ -1,5 +1,6 @@ import * as k8s from "@pulumi/kubernetes"; import * as pulumi from "@pulumi/pulumi"; +import * as k8sClient from "@kubernetes/client-node"; // PrometheusOperatorArgs are the options to configure on the CoreOS // PrometheusOperator. @@ -9,8 +10,9 @@ interface PrometheusOperatorArgs { } // PrometheusOperator implements the CoreOS Prometheus Operator. -export class PrometheusOperator extends pulumi.ComponentResource { +class PrometheusOperator extends pulumi.ComponentResource { public readonly configFile: k8s.yaml.ConfigFile; + public readonly serviceMonitorReady: pulumi.Output; constructor( name: string, args: PrometheusOperatorArgs, @@ -33,6 +35,36 @@ export class PrometheusOperator extends pulumi.ComponentResource { }, ], }, { parent: this }); + + // Wait for the CRD to be established. + // Use the k8s JS client (https://github.com/kubernetes-client/javascript) + // to retrieve resource created by the operator as a workaround for: + // https://github.com/pulumi/pulumi-kubernetes/issues/1056 + this.serviceMonitorReady = pulumi.output(this.configFile).apply(async () => { + const kc = new k8sClient.KubeConfig(); + kc.loadFromDefault(); + const kApi = kc.makeApiClient(k8sClient.ApiextensionsV1Api) + + for (let i = 0; i < 20; i++){ + try { + const crd = await kApi.readCustomResourceDefinition("servicemonitors.monitoring.coreos.com"); + let conditions = crd?.body?.status?.conditions; + if (conditions) { + for (let c of conditions) { + if (c.type == "Established" && c.status == "True") { + console.log("ServiceMonitor CRD is ready"); + return true; + } + } + } + break; + } catch(e) { + console.log(`Waiting for ServiceMonitor CRD to be ready (${i+1})`) + await new Promise(resolve => setTimeout(resolve, 3 * 1000)); + } + } + return false; + }) } } @@ -42,37 +74,41 @@ const prometheusOperator = new PrometheusOperator("prometheus", { }); // Create the Prometheus Operator ServiceMonitor. -const myMonitoring = new k8s.apiextensions.CustomResource('my-monitoring', { - apiVersion: 'monitoring.coreos.com/v1', - kind: 'ServiceMonitor', - spec: { - selector: { - matchLabels: { app: 'my-app' }, - }, - endpoints: [ - { - port: 'http', - interval: '65s', - // start with the following - relabelings: [ +const myMonitoring = prometheusOperator.serviceMonitorReady.apply(ready => { + if (ready){ + return new k8s.apiextensions.CustomResource('my-monitoring', { + apiVersion: 'monitoring.coreos.com/v1', + kind: 'ServiceMonitor', + spec: { + selector: { + matchLabels: { app: 'my-app' }, + }, + endpoints: [ { - regex: '(.*)', - targetLabel: 'stackdriver', - replacement: 'true', - action: 'replace' - } + port: 'http', + interval: '65s', + // start with the following + relabelings: [ + { + regex: '(.*)', + targetLabel: 'stackdriver', + replacement: 'true', + action: 'replace' + } + ], + // try to add the following in replacement of above in steps/step1.ts + // metricRelabelings: [ + // { + // sourceLabels: ['__name__'], + // regex: 'typhoon_(.*)', + // targetLabel: 'stackdriver', + // replacement: 'true', + // action: 'replace' + // } + // ] + }, ], - // try to add the following in replacement of above in steps/step1.ts - // metricRelabelings: [ - // { - // sourceLabels: ['__name__'], - // regex: 'typhoon_(.*)', - // targetLabel: 'stackdriver', - // replacement: 'true', - // action: 'replace' - // } - // ] }, - ], - }, -}, {dependsOn: prometheusOperator}); + }); + } +}); diff --git a/tests/examples/prometheus-operator/package.json b/tests/examples/prometheus-operator/package.json index d767d88adf..635e7f3bdc 100644 --- a/tests/examples/prometheus-operator/package.json +++ b/tests/examples/prometheus-operator/package.json @@ -2,6 +2,8 @@ "name": "prometheus-operator", "version": "0.1.0", "dependencies": { + "@kubernetes/client-node": "latest", + "@pulumi/kubernetes": "^2.3.1", "@pulumi/pulumi": "latest" }, "peerDependencies": { diff --git a/tests/examples/prometheus-operator/step1/index.ts b/tests/examples/prometheus-operator/step1/index.ts index 23f08cc6ca..8bacdd87dc 100644 --- a/tests/examples/prometheus-operator/step1/index.ts +++ b/tests/examples/prometheus-operator/step1/index.ts @@ -1,5 +1,6 @@ import * as k8s from "@pulumi/kubernetes"; import * as pulumi from "@pulumi/pulumi"; +import * as k8sClient from "@kubernetes/client-node"; // PrometheusOperatorArgs are the options to configure on the CoreOS // PrometheusOperator. @@ -9,8 +10,9 @@ interface PrometheusOperatorArgs { } // PrometheusOperator implements the CoreOS Prometheus Operator. -export class PrometheusOperator extends pulumi.ComponentResource { +class PrometheusOperator extends pulumi.ComponentResource { public readonly configFile: k8s.yaml.ConfigFile; + public readonly serviceMonitorReady: pulumi.Output; constructor( name: string, args: PrometheusOperatorArgs, @@ -33,6 +35,36 @@ export class PrometheusOperator extends pulumi.ComponentResource { }, ], }, { parent: this }); + + // Wait for the CRD to be established. + // Use the k8s JS client (https://github.com/kubernetes-client/javascript) + // to retrieve resource created by the operator as a workaround for: + // https://github.com/pulumi/pulumi-kubernetes/issues/1056 + this.serviceMonitorReady = pulumi.output(this.configFile).apply(async () => { + const kc = new k8sClient.KubeConfig(); + kc.loadFromDefault(); + const kApi = kc.makeApiClient(k8sClient.ApiextensionsV1Api) + + for (let i = 0; i < 20; i++){ + try { + const crd = await kApi.readCustomResourceDefinition("servicemonitors.monitoring.coreos.com"); + let conditions = crd?.body?.status?.conditions; + if (conditions) { + for (let c of conditions) { + if (c.type == "Established" && c.status == "True") { + console.log("ServiceMonitor CRD is ready"); + return true; + } + } + } + break; + } catch(e) { + console.log(`Waiting for ServiceMonitor CRD to be ready (${i+1})`) + await new Promise(resolve => setTimeout(resolve, 3 * 1000)); + } + } + return false; + }) } } @@ -42,37 +74,41 @@ const prometheusOperator = new PrometheusOperator("prometheus", { }); // Create the Prometheus Operator ServiceMonitor. -const myMonitoring = new k8s.apiextensions.CustomResource('my-monitoring', { - apiVersion: 'monitoring.coreos.com/v1', - kind: 'ServiceMonitor', - spec: { - selector: { - matchLabels: { app: 'my-app' }, - }, - endpoints: [ - { - port: 'http', - interval: '65s', - // removing the following in index.ts in favor of below - // relabelings: [ - // { - // regex: '(.*)', - // targetLabel: 'stackdriver', - // replacement: 'true', - // action: 'replace' - // } - // ], - // add the following in replacement of above in index.ts - metricRelabelings: [ +const myMonitoring = prometheusOperator.serviceMonitorReady.apply(ready => { + if (ready){ + return new k8s.apiextensions.CustomResource('my-monitoring', { + apiVersion: 'monitoring.coreos.com/v1', + kind: 'ServiceMonitor', + spec: { + selector: { + matchLabels: { app: 'my-app' }, + }, + endpoints: [ { - sourceLabels: ['__name__'], - regex: 'typhoon_(.*)', - targetLabel: 'stackdriver', - replacement: 'true', - action: 'replace' - } - ] + port: 'http', + interval: '65s', + // removing the following in index.ts in favor of below + // relabelings: [ + // { + // regex: '(.*)', + // targetLabel: 'stackdriver', + // replacement: 'true', + // action: 'replace' + // } + // ], + // add the following in replacement of above in index.ts + metricRelabelings: [ + { + sourceLabels: ['__name__'], + regex: 'typhoon_(.*)', + targetLabel: 'stackdriver', + replacement: 'true', + action: 'replace' + } + ] + }, + ], }, - ], - }, -}, {dependsOn: prometheusOperator}); + }); + } +})