From 245768150ef5c627964c3dc14190019f423c72d1 Mon Sep 17 00:00:00 2001 From: Anshuman Date: Wed, 1 Jan 2025 15:28:41 +0530 Subject: [PATCH] Review changes --- cmd/main.go | 11 ++ config/default/manager_metrics_service.yaml | 2 +- config/rbac/auth_proxy_client_binding.yaml | 2 +- test/e2e/e2e_test.go | 151 ++++++++++++++++++++ test/testutils/util.go | 48 +++++++ 5 files changed, 212 insertions(+), 2 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 937a913c..3f217acc 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -17,6 +17,7 @@ limitations under the License. package main import ( + "crypto/tls" "flag" "os" "time" @@ -106,6 +107,15 @@ func main() { kubeConfig.Burst = burst namespace := utils.GetOperatorNamespace() + // Disabling http/2 to prevent being vulnerable to the HTTP/2 Stream Cancellation and + // Rapid Reset CVEs. For more information see: + // - https://github.com/advisories/GHSA-qppj-fm5r-hxr3 + // - https://github.com/advisories/GHSA-4374-p667-p6c8 + disableHTTP2 := func(c *tls.Config) { + setupLog.Info("disabling http/2") + c.NextProtos = []string{"http/1.1"} + } + // Metrics endpoint is enabled in 'config/default/kustomization.yaml'. The Metrics options configure the server. // More info: // - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.19.1/pkg/metrics/server @@ -114,6 +124,7 @@ func main() { BindAddress: metricsAddr, SecureServing: true, FilterProvider: filters.WithAuthenticationAndAuthorization, + TLSOpts: []func(*tls.Config){disableHTTP2}, } mgr, err := ctrl.NewManager(kubeConfig, ctrl.Options{ diff --git a/config/default/manager_metrics_service.yaml b/config/default/manager_metrics_service.yaml index ff9ecfa5..1b137e2c 100644 --- a/config/default/manager_metrics_service.yaml +++ b/config/default/manager_metrics_service.yaml @@ -14,4 +14,4 @@ spec: protocol: TCP targetPort: 8443 selector: - control-plane: controller-manager \ No newline at end of file + control-plane: controller-manager diff --git a/config/rbac/auth_proxy_client_binding.yaml b/config/rbac/auth_proxy_client_binding.yaml index 2d14de4d..f7d0f2fc 100644 --- a/config/rbac/auth_proxy_client_binding.yaml +++ b/config/rbac/auth_proxy_client_binding.yaml @@ -9,4 +9,4 @@ roleRef: subjects: - kind: ServiceAccount name: controller-manager - namespace: system \ No newline at end of file + namespace: system diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 81416c9a..5303e482 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -16,6 +16,13 @@ limitations under the License. package e2e import ( + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "time" + "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" @@ -301,4 +308,148 @@ var _ = ginkgo.Describe("leaderWorkerSet e2e tests", func() { return numberOfPodsInCommon, nil }, timeout, interval).Should(gomega.Equal(0)) }) + + // metricsRoleBindingName := "lws-metrics-reader-rolebinding" + serviceAccountName := "lws-controller-manager" + metricsServiceName := "lws-controller-manager-metrics-service" + namespace := "lws-system" + var controllerPodName string + + ginkgo.It("should ensure the metrics endpoint is serving metrics", func() { + + ginkgo.By("fetching the controller pod name") + cmd := exec.Command("kubectl", "get", + "pods", "-l", "control-plane=controller-manager", + "-n", namespace, + "-o", "go-template={{ range .items }}"+ + "{{ if not .metadata.deletionTimestamp }}"+ + "{{ .metadata.name }}"+ + "{{ \"\\n\" }}{{ end }}{{ end }}", + ) + podOutput, err := testutils.Run(cmd) + gomega.Expect(err).NotTo(gomega.HaveOccurred(), "Failed to retrieve controller-manager pod information") + podNames := testutils.GetNonEmptyLines(podOutput) + controllerPodName = podNames[0] + + ginkgo.By("validating that the metrics service is available") + cmd = exec.Command("kubectl", "get", "service", metricsServiceName, "-n", namespace) + _, err = testutils.Run(cmd) + gomega.Expect(err).NotTo(gomega.HaveOccurred(), "Metrics service should exist") + + ginkgo.By("getting the service account token") + token, err := serviceAccountToken(serviceAccountName, namespace) + gomega.Expect(err).NotTo(gomega.HaveOccurred()) + gomega.Expect(token).NotTo(gomega.BeEmpty()) + + ginkgo.By("waiting for the metrics endpoint to be ready") + verifyMetricsEndpointReady := func(g gomega.Gomega) { + cmd := exec.Command("kubectl", "get", "endpoints", metricsServiceName, "-n", namespace) + g.Expect(err).NotTo(gomega.HaveOccurred()) + output, err := testutils.Run(cmd) + g.Expect(err).Should(gomega.BeNil()) + g.Expect(output).To(gomega.ContainSubstring("8080"), "Metrics endpoint is not ready") + } + gomega.Eventually(verifyMetricsEndpointReady).Should(gomega.Succeed()) + + ginkgo.By("verifying that the controller manager is serving the metrics server") + verifyMetricsServerStarted := func(g gomega.Gomega) { + cmd := exec.Command("kubectl", "logs", controllerPodName, "-n", namespace) + output, err := testutils.Run(cmd) + g.Expect(err).NotTo(gomega.HaveOccurred()) + g.Expect(output).To(gomega.ContainSubstring("controller-runtime.metrics\tServing metrics server"), + "Metrics server not yet started") + } + gomega.Eventually(verifyMetricsServerStarted).Should(gomega.Succeed()) + + ginkgo.By("creating the curl-metrics pod to access the metrics endpoint") + cmd = exec.Command("kubectl", "run", "curl-metrics", "--restart=Never", + "--namespace", namespace, + "--image=curlimages/curl:7.78.0", + "--", "/bin/sh", "-c", fmt.Sprintf( + "curl -v -k -H 'Authorization: Bearer %s' https://%s.%s.svc.cluster.local:8080/metrics", + token, metricsServiceName, namespace)) + _, err = testutils.Run(cmd) + gomega.Expect(err).NotTo(gomega.HaveOccurred(), "Failed to create curl-metrics pod") + + ginkgo.By("waiting for the curl-metrics pod to complete.") + verifyCurlUp := func(g gomega.Gomega) { + cmd := exec.Command("kubectl", "get", "pods", "curl-metrics", + "-o", "jsonpath={.status.phase}", + "-n", namespace) + output, err := testutils.Run(cmd) + g.Expect(err).NotTo(gomega.HaveOccurred()) + g.Expect(output).To(gomega.Equal("Succeeded"), "curl pod in wrong status") + } + gomega.Eventually(verifyCurlUp, 5*time.Minute).Should(gomega.Succeed()) + + ginkgo.By("getting the metrics by checking curl-metrics logs") + metricsOutput := getMetricsOutput(namespace) + gomega.Expect(metricsOutput).To(gomega.ContainSubstring( + "controller_runtime_reconcile_total", + )) + + ginkgo.By("cleaning up the curl-metrics pod") + cmd = exec.Command("kubectl", "delete", "pod", "-n", namespace, "curl-metrics") + _, err = testutils.Run(cmd) + gomega.Expect(err).To(gomega.BeNil()) + }) }) + +// tokenRequest is a simplified representation of the Kubernetes TokenRequest API response, +// containing only the token field that we need to extract. +type tokenRequest struct { + Status struct { + Token string `json:"token"` + } `json:"status"` +} + +// serviceAccountToken returns a token for the specified service account in the given namespace. +// It uses the Kubernetes TokenRequest API to generate a token by directly sending a request +// and parsing the resulting token from the API response. +func serviceAccountToken(serviceAccountName, namespace string) (string, error) { + const tokenRequestRawString = `{ + "apiVersion": "authentication.k8s.io/v1", + "kind": "TokenRequest" + }` + + // Temporary file to store the token request + secretName := fmt.Sprintf("%s-token-request", serviceAccountName) + tokenRequestFile := filepath.Join("/tmp", secretName) + err := os.WriteFile(tokenRequestFile, []byte(tokenRequestRawString), os.FileMode(0o644)) + if err != nil { + return "", err + } + + var out string + verifyTokenCreation := func(g gomega.Gomega) { + // Execute kubectl command to create the token + cmd := exec.Command("kubectl", "create", "--raw", fmt.Sprintf( + "/api/v1/namespaces/%s/serviceaccounts/%s/token", + namespace, + serviceAccountName, + ), "-f", tokenRequestFile) + + output, err := cmd.CombinedOutput() + g.Expect(err).NotTo(gomega.HaveOccurred()) + + // Parse the JSON output to extract the token + var token tokenRequest + err = json.Unmarshal(output, &token) + g.Expect(err).NotTo(gomega.HaveOccurred()) + + out = token.Status.Token + } + gomega.Eventually(verifyTokenCreation).Should(gomega.Succeed()) + + return out, err +} + +// getMetricsOutput retrieves and returns the logs from the curl pod used to access the metrics endpoint. +func getMetricsOutput(namespace string) string { + ginkgo.By("getting the curl-metrics logs") + cmd := exec.Command("kubectl", "logs", "curl-metrics", "-n", namespace) + metricsOutput, err := testutils.Run(cmd) + gomega.Expect(err).NotTo(gomega.HaveOccurred(), "Failed to retrieve logs from curl pod") + gomega.Expect(metricsOutput).To(gomega.ContainSubstring("< HTTP/1.1 200 OK")) + return metricsOutput +} diff --git a/test/testutils/util.go b/test/testutils/util.go index 8c2c0060..b86185c6 100644 --- a/test/testutils/util.go +++ b/test/testutils/util.go @@ -18,8 +18,12 @@ import ( "context" "errors" "fmt" + "os" + "os/exec" "strconv" + "strings" + "github.com/onsi/ginkgo/v2" "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -564,3 +568,47 @@ func deleteWorkerStatefulSetIfExists(ctx context.Context, k8sClient client.Clien return k8sClient.Delete(ctx, &sts) }, Timeout, Interval).Should(gomega.Succeed()) } + +// GetProjectDir will return the directory where the project is +func GetProjectDir() (string, error) { + wd, err := os.Getwd() + if err != nil { + return wd, err + } + wd = strings.Replace(wd, "/test/e2e", "", -1) + return wd, nil +} + +// Run executes the provided command within this context +func Run(cmd *exec.Cmd) (string, error) { + dir, _ := GetProjectDir() + cmd.Dir = dir + + if err := os.Chdir(cmd.Dir); err != nil { + _, _ = fmt.Fprintf(ginkgo.GinkgoWriter, "chdir dir: %s\n", err) + } + + cmd.Env = append(os.Environ(), "GO111MODULE=on") + command := strings.Join(cmd.Args, " ") + _, _ = fmt.Fprintf(ginkgo.GinkgoWriter, "running: %s\n", command) + output, err := cmd.CombinedOutput() + if err != nil { + return string(output), fmt.Errorf("%s failed with error: (%v) %s", command, err, string(output)) + } + + return string(output), nil +} + +// GetNonEmptyLines converts given command output string into individual objects +// according to line breakers, and ignores the empty elements in it. +func GetNonEmptyLines(output string) []string { + var res []string + elements := strings.Split(output, "\n") + for _, element := range elements { + if element != "" { + res = append(res, element) + } + } + + return res +}