Add TLS insecure skip verify e2e tests

e2e/authorization_test.go

@@ -0,0 +1,164 @@
+// Copyright 2023 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package e2e
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/GoogleCloudPlatform/prometheus-engine/e2e/kubeutil"
+	"github.com/GoogleCloudPlatform/prometheus-engine/e2e/operatorutil"
+	"github.com/GoogleCloudPlatform/prometheus-engine/pkg/operator"
+	monitoringv1 "github.com/GoogleCloudPlatform/prometheus-engine/pkg/operator/apis/monitoring/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+func TestTLS(t *testing.T) {
+	t.Parallel()
+	tctx := newOperatorContext(t)
+	ctx := context.Background()
+
+	tctx.createOperatorConfigFrom(ctx, monitoringv1.OperatorConfig{
+		Features: monitoringv1.OperatorFeatures{
+			TargetStatus: monitoringv1.TargetStatusSpec{
+				Enabled: true,
+			},
+		},
+	})
+
+	c := tctx.Client()
+	const appName = "tls-insecure"
+	deployment, err := operatorutil.SyntheticAppDeploy(ctx, c, tctx.namespace, appName, []string{
+		"--tls-create-self-signed=true",
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := kubeutil.WaitForDeploymentReady(ctx, c, tctx.namespace, appName); err != nil {
+		kubeutil.DeploymentDebug(tctx.T, ctx, tctx.RestConfig(), tctx.Client(), tctx.namespace, appName)
+		t.Fatalf("failed to start app: %s", err)
+	}
+
+	tctx.Run("tls-missing-config", tctx.subtest(func(ctx context.Context, t *OperatorContext) {
+		t.Parallel()
+
+		pm := &monitoringv1.PodMonitoring{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "collector-tls-missing-config",
+				Namespace: t.namespace,
+			},
+			Spec: monitoringv1.PodMonitoringSpec{
+				Selector: metav1.LabelSelector{
+					MatchLabels: deployment.Spec.Template.Labels,
+				},
+				Endpoints: []monitoringv1.ScrapeEndpoint{
+					{
+						Port:     intstr.FromString(operatorutil.SyntheticAppPortName),
+						Scheme:   "https",
+						Interval: "5s",
+					},
+				},
+			},
+		}
+		if err := t.Client().Create(ctx, pm); err != nil {
+			t.Fatalf("create collector PodMonitoring: %s", err)
+		}
+
+		if err := operatorutil.WaitForPodMonitoringReady(ctx, t.Client(), pm, true); err != nil {
+			kubeutil.DaemonSetDebug(tctx.T, ctx, tctx.RestConfig(), tctx.Client(), tctx.namespace, operator.NameCollector)
+			t.Fatalf("collector not ready: %s", err)
+		}
+
+		var err error
+		if pollErr := wait.Poll(5*time.Second, 3*time.Minute, func() (bool, error) {
+			if err = t.Client().Get(ctx, client.ObjectKeyFromObject(pm), pm); err != nil {
+				return false, nil
+			}
+
+			const expected = "tls: failed to verify certificate: x509: certificate signed by unknown authority"
+			err = operatorutil.IsPodMonitoringScrapeEndpointFailure(pm, operatorutil.SyntheticAppPortName, func(message string) error {
+				if !strings.HasSuffix(message, expected) {
+					return fmt.Errorf("expected %q", expected)
+				}
+				return nil
+			})
+			return err == nil, nil
+		}); pollErr != nil {
+			if errors.Is(pollErr, wait.ErrWaitTimeout) {
+				pollErr = err
+			}
+			kubeutil.DaemonSetDebug(tctx.T, ctx, tctx.RestConfig(), tctx.Client(), tctx.namespace, operator.NameCollector)
+			t.Fatalf("scrape endpoint expected failure: %s", pollErr)
+		}
+	}))
+
+	tctx.Run("tls-insecure", tctx.subtest(func(ctx context.Context, t *OperatorContext) {
+		t.Parallel()
+
+		pm := &monitoringv1.PodMonitoring{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "collector-tls-insecure",
+				Namespace: t.namespace,
+			},
+			Spec: monitoringv1.PodMonitoringSpec{
+				Selector: metav1.LabelSelector{
+					MatchLabels: deployment.Spec.Template.Labels,
+				},
+				Endpoints: []monitoringv1.ScrapeEndpoint{
+					{
+						Port:     intstr.FromString(operatorutil.SyntheticAppPortName),
+						Scheme:   "https",
+						Interval: "5s",
+						HTTPClientConfig: monitoringv1.HTTPClientConfig{
+							TLS: &monitoringv1.TLS{
+								InsecureSkipVerify: true,
+							},
+						},
+					},
+				},
+			},
+		}
+		if err := t.Client().Create(ctx, pm); err != nil {
+			t.Fatalf("create collector: %s", err)
+		}
+
+		if err := operatorutil.WaitForPodMonitoringReady(ctx, t.Client(), pm, true); err != nil {
+			kubeutil.DaemonSetDebug(tctx.T, ctx, tctx.RestConfig(), tctx.Client(), tctx.namespace, operator.NameCollector)
+			t.Errorf("collector not ready: %s", err)
+		}
+
+		var err error
+		if pollErr := wait.Poll(5*time.Second, 3*time.Minute, func() (bool, error) {
+			if err = t.Client().Get(ctx, client.ObjectKeyFromObject(pm), pm); err != nil {
+				return false, nil
+			}
+			err = operatorutil.IsPodMonitoringScrapeEndpointSuccess(pm, operatorutil.SyntheticAppPortName)
+			return err == nil, nil
+		}); pollErr != nil {
+			if errors.Is(pollErr, wait.ErrWaitTimeout) {
+				pollErr = err
+			}
+			kubeutil.DaemonSetDebug(tctx.T, ctx, tctx.RestConfig(), tctx.Client(), tctx.namespace, operator.NameCollector)
+			t.Fatalf("scrape endpoint expected success: %s", pollErr)
+		}
+	}))
+}

e2e/collector_test.go

@@ -25,6 +25,7 @@ import (
 	gcm "cloud.google.com/go/monitoring/apiv3/v2"
 	gcmpb "cloud.google.com/go/monitoring/apiv3/v2/monitoringpb"
 	"github.com/GoogleCloudPlatform/prometheus-engine/e2e/kubeutil"
+	"github.com/GoogleCloudPlatform/prometheus-engine/e2e/operatorutil"
 	"github.com/google/go-cmp/cmp"
 	"google.golang.org/api/iterator"
 	"google.golang.org/protobuf/types/known/timestamppb"
@@ -97,13 +98,14 @@ func TestCollector(t *testing.T) {
 			})
 		}))
 
-		const appName = "tls-insecure"
-		deployment, err := SyntheticAppDeploy(ctx, tctx.Client(), tctx.namespace, appName, []string{})
+		const appName = "collector-synthetic"
+		deployment, err := operatorutil.SyntheticAppDeploy(ctx, tctx.Client(), tctx.namespace, appName, []string{})
 		if err != nil {
 			tctx.Fatal(err)
 		}
 
 		if err := kubeutil.WaitForDeploymentReady(ctx, tctx.Client(), tctx.namespace, appName); err != nil {
+			kubeutil.DeploymentDebug(tctx.T, ctx, tctx.RestConfig(), tctx.Client(), tctx.namespace, appName)
 			tctx.Fatalf("failed to start app: %s", err)
 		}
 		t.Run("synthetic-podmonitoring", tctx.subtest(func(ctx context.Context, t *OperatorContext) {
@@ -119,7 +121,8 @@ func TestCollector(t *testing.T) {
 					},
 					Endpoints: []monitoringv1.ScrapeEndpoint{
 						{
-							Port: intstr.FromString(SyntheticAppPortName),
+							Port:     intstr.FromString(operatorutil.SyntheticAppPortName),
+							Interval: "5s",
 						},
 					},
 				},
@@ -137,7 +140,8 @@ func TestCollector(t *testing.T) {
 					},
 					Endpoints: []monitoringv1.ScrapeEndpoint{
 						{
-							Port: intstr.FromString(SyntheticAppPortName),
+							Port:     intstr.FromString(operatorutil.SyntheticAppPortName),
+							Interval: "5s",
 						},
 					},
 				},
@@ -262,21 +266,22 @@ func testCollector(ctx context.Context, t *OperatorContext, pm monitoringv1.PodM
 	}
 	t.Logf("Waiting for %q to be processed", pm.GetName())
 
-	if err := WaitForPodMonitoringReady(ctx, t.Client(), pm, true); err != nil {
+	if err := operatorutil.WaitForPodMonitoringReady(ctx, t.Client(), pm, true); err != nil {
 		t.Errorf("unable to validate status: %s", err)
 	}
 
 	var err error
-	if pollErr := wait.Poll(3*time.Second, 2*time.Minute, func() (bool, error) {
+	if pollErr := wait.Poll(3*time.Second, 3*time.Minute, func() (bool, error) {
 		if err = t.Client().Get(ctx, client.ObjectKeyFromObject(pm), pm); err != nil {
 			return false, nil
 		}
-		err = IsPodMonitoringSuccess(pm, true)
+		err = operatorutil.IsPodMonitoringSuccess(pm, true)
 		return err == nil, nil
 	}); pollErr != nil {
 		if errors.Is(pollErr, wait.ErrWaitTimeout) && err != nil {
 			pollErr = err
 		}
+		kubeutil.DaemonSetDebug(t.T, ctx, t.RestConfig(), t.Client(), t.namespace, operator.NameCollector)
 		t.Errorf("status does not indicate success: %s", pollErr)
 	}
 

e2e/kubeclient.go → e2e/kubeutil/client.go

@@ -16,7 +16,7 @@
 // To make tests simple and fast, the test suite runs the operator internally. The CRDs
 // are expected to be installed out of band (along with the operator deployment itself in
 // a real world setup).
-package e2e
+package kubeutil
 
 import (
 	"context"

e2e/kubeutil/container.go

@@ -0,0 +1,113 @@
+// Copyright 2023 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package e2e contains tests that validate the behavior of gmp-operator against a cluster.
+// To make tests simple and fast, the test suite runs the operator internally. The CRDs
+// are expected to be installed out of band (along with the operator deployment itself in
+// a real world setup).
+package kubeutil
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strings"
+	"testing"
+
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/client-go/rest"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+func containerStateString(state *corev1.ContainerState) string {
+	if state.Waiting != nil {
+		return fmt.Sprintf("waiting due to %s", state.Waiting.Reason)
+	}
+	if state.Terminated != nil {
+		return fmt.Sprintf("terminated due to %s", state.Terminated.Reason)
+	}
+	return "running"
+}
+
+func containerPods(ctx context.Context, kubeClient client.Client, o client.Object) ([]corev1.Pod, error) {
+	switch o := o.(type) {
+	case *appsv1.Deployment:
+		return DeploymentPods(ctx, kubeClient, o)
+	case *appsv1.DaemonSet:
+		return DaemonSetPods(ctx, kubeClient, o)
+	default:
+		return nil, errors.New("invalid object type")
+	}
+}
+
+func containerDebug(t testing.TB, ctx context.Context, restConfig *rest.Config, kubeClient client.Client, gvk schema.GroupVersionKind, o client.Object, typeName string) {
+	namespace := o.GetNamespace()
+	name := o.GetName()
+	t.Logf("%s %s/%s events:", typeName, namespace, name)
+	events, err := Events(ctx, kubeClient, gvk, namespace, name)
+	if err != nil {
+		t.Errorf("unable to get %s %s/%s events: %s", typeName, namespace, name, err)
+	} else {
+		t.Log(strings.Join(events, "\n"))
+	}
+
+	if err := kubeClient.Get(ctx, client.ObjectKeyFromObject(o), o); err != nil {
+		t.Fatalf("unable to get deployment %s/%s: %s", namespace, name, err)
+	}
+
+	// Best effort to find a problematic pod container.
+	pods, err := containerPods(ctx, kubeClient, o)
+	if err != nil {
+		t.Errorf("unable to get container pods")
+	}
+	if len(pods) == 0 {
+		t.Log("no pods found for this deployment")
+		return
+	}
+
+	showPodLogs := func(pod *corev1.Pod, container string) {
+		t.Logf("sample pod %s/%s container %s logs:", pod.Namespace, pod.Name, container)
+		logs, err := PodLogs(ctx, restConfig, pod.Namespace, pod.Name, container)
+		if err != nil {
+			t.Errorf("unable to get pod %s/%s container %s logs: %s", pod.Namespace, pod.Name, container, err)
+		} else {
+			t.Log(logs)
+		}
+	}
+
+	for _, pod := range pods {
+		found := false
+		for _, status := range pod.Status.ContainerStatuses {
+			// The pod is crash-looping.
+			if status.RestartCount > 1 {
+				found = true
+				showPodLogs(&pod, status.Name)
+			}
+		}
+		// Not perfect, but hopefully we found an issue.
+		if found {
+			return
+		}
+	}
+
+	// Worse case, let's just show the first one.
+	t.Log("found no crash-looping pods -- showing logs of first pod")
+	for _, pod := range pods {
+		for _, status := range pod.Status.ContainerStatuses {
+			showPodLogs(&pod, status.Name)
+		}
+	}
+}