From 51316a3625d84fd3d9e265b580f58685d89d19a4 Mon Sep 17 00:00:00 2001 From: Kanha gupta Date: Tue, 30 Apr 2024 03:37:03 +0530 Subject: [PATCH 01/12] Pre-installation testing framework Signed-off-by: Kanha gupta --- .github/workflows/kind.yml | 9 +- pkg/antctl/antctl.go | 7 + pkg/antctl/raw/check/cluster/command.go | 209 ++++++++++++++++++ .../check/cluster/test_checkCNIExistence.go | 60 +++++ .../test_checkcontrolplaneavailability.go | 52 +++++ .../raw/check/cluster/test_checkk8sversion.go | 48 ++++ .../check/cluster/test_checkovsloadable.go | 64 ++++++ pkg/antctl/raw/check/installation/command.go | 91 +------- pkg/antctl/raw/check/util.go | 89 ++++++++ 9 files changed, 540 insertions(+), 89 deletions(-) create mode 100644 pkg/antctl/raw/check/cluster/command.go create mode 100644 pkg/antctl/raw/check/cluster/test_checkCNIExistence.go create mode 100644 pkg/antctl/raw/check/cluster/test_checkcontrolplaneavailability.go create mode 100644 pkg/antctl/raw/check/cluster/test_checkk8sversion.go create mode 100644 pkg/antctl/raw/check/cluster/test_checkovsloadable.go diff --git a/.github/workflows/kind.yml b/.github/workflows/kind.yml index 845d5e3a9ca..08bd2934b81 100644 --- a/.github/workflows/kind.yml +++ b/.github/workflows/kind.yml @@ -772,13 +772,16 @@ jobs: - name: Create Kind Cluster run: | kind create cluster --config ci/kind/config-3nodes.yml + - name: Build antctl binary + run: | + make antctl-linux + - name: Run Pre-installation checks + run: | + ./bin/antctl-linux check cluster - name: Load Docker images and deploy Antrea run: | kind load docker-image antrea/antrea-controller-ubuntu-coverage:latest antrea/antrea-agent-ubuntu-coverage:latest kubectl apply -f build/yamls/antrea.yml - - name: Build antctl binary - run: | - make antctl-linux - name: Run antctl command run: | ./bin/antctl-linux check installation diff --git a/pkg/antctl/antctl.go b/pkg/antctl/antctl.go index 2addc2d44e9..946dbb3d53e 100644 --- a/pkg/antctl/antctl.go +++ b/pkg/antctl/antctl.go @@ -19,6 +19,7 @@ import ( agentapis "antrea.io/antrea/pkg/agent/apis" fallbackversion "antrea.io/antrea/pkg/antctl/fallback/version" + checkcluster "antrea.io/antrea/pkg/antctl/raw/check/cluster" checkinstallation "antrea.io/antrea/pkg/antctl/raw/check/installation" "antrea.io/antrea/pkg/antctl/raw/featuregates" "antrea.io/antrea/pkg/antctl/raw/multicluster" @@ -640,6 +641,12 @@ $ antctl get podmulticaststats pod -n namespace`, supportController: false, commandGroup: check, }, + { + cobraCommand: checkcluster.Command(), + supportAgent: false, + supportController: false, + commandGroup: check, + }, { cobraCommand: supportbundle.Command, supportAgent: true, diff --git a/pkg/antctl/raw/check/cluster/command.go b/pkg/antctl/raw/check/cluster/command.go new file mode 100644 index 00000000000..b8c5a3582fc --- /dev/null +++ b/pkg/antctl/raw/check/cluster/command.go @@ -0,0 +1,209 @@ +// Copyright 2024 Antrea Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cluster + +import ( + "context" + "fmt" + "os" + "time" + + "github.com/spf13/cobra" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/utils/ptr" + + "antrea.io/antrea/pkg/antctl/raw/check" +) + +func Command() *cobra.Command { + o := newOptions() + command := &cobra.Command{ + Use: "cluster", + Short: "Runs pre installation checks", + RunE: func(cmd *cobra.Command, args []string) error { + return Run(o) + }, + } + command.Flags().StringVarP(&o.antreaNamespace, "Namespace", "n", o.antreaNamespace, "Configure Namespace in which Antrea is running") + return command +} + +type options struct { + antreaNamespace string +} + +func newOptions() *options { + return &options{ + antreaNamespace: "kube-system", + } +} + +const ( + antreaNamespace = "kube-system" + deploymentName = "cluster-check" + podReadyTimeout = 1 * time.Minute +) + +type Test interface { + Run(ctx context.Context, testContext *testContext) error +} + +var testsRegistry = make(map[string]Test) + +func RegisterTest(name string, test Test) { + testsRegistry[name] = test +} + +type testContext struct { + client kubernetes.Interface + config *rest.Config + clusterName string + antreaNamespace string +} + +func Run(o *options) error { + client, config, clusterName, err := check.NewClient() + if err != nil { + return fmt.Errorf("unable to create Kubernetes client: %s", err) + } + ctx := context.Background() + testContext := NewTestContext(client, config, clusterName, o) + if err := testContext.setup(ctx); err != nil { + return err + } + for name, test := range testsRegistry { + testContext.Header("Running test: %s", name) + if err := test.Run(ctx, testContext); err != nil { + testContext.Header("Test %s failed: %s", name, err) + } else { + testContext.Header("Test %s passed", name) + } + } + testContext.Log("Test finished") + testContext.teardown(ctx, deploymentName, antreaNamespace) + return nil +} + +func (t *testContext) setup(ctx context.Context) error { + deployment := check.NewDeployment(check.DeploymentParameters{ + Name: deploymentName, + Image: "antrea/antrea-agent-ubuntu:latest", + Replicas: 1, + Command: []string{"sleep", "infinity"}, + Labels: map[string]string{"app": "cluster-check"}, + HostNetwork: true, + VolumeMounts: []corev1.VolumeMount{ + {Name: "cni-conf", MountPath: "/etc/cni/net.d"}, + {Name: "lib-modules", MountPath: "/lib/modules"}, + }, + Tolerations: []corev1.Toleration{ + { + Key: "node-role.kubernetes.io/control-plane", + Operator: "Exists", + Effect: "NoSchedule", + }, + { + Key: "node-role.kubernetes.io/master", + Operator: "Exists", + Effect: "NoSchedule", + }, + { + Key: "node.kubernetes.io/not-ready", + Operator: "Exists", + Effect: "NoSchedule", + }, + }, + Volumes: []corev1.Volume{ + { + Name: "cni-conf", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/etc/cni/net.d", + }, + }, + }, + { + Name: "lib-modules", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/lib/modules", + Type: ptr.To(corev1.HostPathType("Directory")), + }, + }, + }, + }, + }) + + t.Log("Creating Deployment") + _, err := t.client.AppsV1().Deployments(antreaNamespace).Create(ctx, deployment, metav1.CreateOptions{}) + if err != nil { + return fmt.Errorf("unable to create Deployment: %w", err) + } + + t.Log("Waiting for Deployment to become ready") + check.WaitForDeploymentsReady(ctx, time.Second, podReadyTimeout, t.client, t.antreaNamespace, t.clusterName, deploymentName) + if err != nil { + return fmt.Errorf("error while waiting for Deployment to become ready: %w", err) + } + return nil +} + +func NewTestContext(client kubernetes.Interface, config *rest.Config, clusterName string, o *options) *testContext { + return &testContext{ + client: client, + config: config, + clusterName: clusterName, + antreaNamespace: o.antreaNamespace, + } +} + +func (t *testContext) teardown(ctx context.Context, deploymentName, namespace string) error { + err := t.client.AppsV1().Deployments(namespace).Delete(ctx, deploymentName, metav1.DeleteOptions{}) + if err != nil { + return err + } + t.Log("Waiting for the deletion of Deployment %s in Namespace %s...", deploymentName, namespace) + err = wait.PollUntilContextTimeout(ctx, 2*time.Second, 1*time.Minute, true, func(ctx context.Context) (bool, error) { + _, err := t.client.AppsV1().Deployments(namespace).Get(ctx, deploymentName, metav1.GetOptions{}) + if errors.IsNotFound(err) { + return true, nil + } + if err != nil { + return false, err + } + return false, nil + }) + if err != nil { + return fmt.Errorf("error waiting for Deployment %s to be deleted in Namespace %s: %w", deploymentName, namespace, err) + } + t.Log("Deployment %s successfully deleted from Namespace %s", deploymentName, namespace) + return nil +} + +func (t *testContext) Log(format string, a ...interface{}) { + fmt.Fprintf(os.Stdout, fmt.Sprintf("[%s] ", t.clusterName)+format+"\n", a...) +} + +func (t *testContext) Header(format string, a ...interface{}) { + t.Log("-------------------------------------------------------------------------------------------") + t.Log(format, a...) + t.Log("-------------------------------------------------------------------------------------------") +} diff --git a/pkg/antctl/raw/check/cluster/test_checkCNIExistence.go b/pkg/antctl/raw/check/cluster/test_checkCNIExistence.go new file mode 100644 index 00000000000..6fd7b8e8799 --- /dev/null +++ b/pkg/antctl/raw/check/cluster/test_checkCNIExistence.go @@ -0,0 +1,60 @@ +// Copyright 2024 Antrea Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cluster + +import ( + "context" + "fmt" + "sort" + "strings" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "antrea.io/antrea/pkg/antctl/raw/check" +) + +type checkCNIExistence struct{} + +func init() { + RegisterTest("Check if another CNI is Present", &checkCNIExistence{}) +} + +func (t *checkCNIExistence) Run(ctx context.Context, testContext *testContext) error { + pods, err := testContext.client.CoreV1().Pods(antreaNamespace).List(ctx, metav1.ListOptions{LabelSelector: "name=cluster-check"}) + if err != nil { + return fmt.Errorf("failed to list Pods: %v", err) + } + testContext.Log("Checking CNI configurations in Pod: %s", pods.Items[0].Name) + command := []string{"ls", "/etc/cni/net.d"} + output, _, err := check.ExecInPod(ctx, testContext.client, testContext.config, antreaNamespace, pods.Items[0].Name, "", command) + if err != nil { + testContext.Log("Failed to execute command in pod: %s, error: %v", pods.Items[0].Name, err) + } + outputStr := strings.TrimSpace(output) + if outputStr == "" { + testContext.Log("No files present in /etc/cni/net.d in pod: %s", pods.Items[0].Name) + } else { + files := strings.Split(outputStr, "\n") + sort.Strings(files) + if len(files) > 0 && files[0] < "10-antrea.conflist" { + testContext.Log("Warning: Another CNI configuration file with higher priority than Antrea's CNI configuration file found: %s", files[0]) + } else if len(files) > 0 && files[0] != "10-antrea.conflist" { + testContext.Log("Warning: Another CNI configuration file found: %s.", files[0]) + } else { + testContext.Log("Antrea's CNI configuration file already present: %s", files[0]) + } + } + return nil +} diff --git a/pkg/antctl/raw/check/cluster/test_checkcontrolplaneavailability.go b/pkg/antctl/raw/check/cluster/test_checkcontrolplaneavailability.go new file mode 100644 index 00000000000..3b2f29851b1 --- /dev/null +++ b/pkg/antctl/raw/check/cluster/test_checkcontrolplaneavailability.go @@ -0,0 +1,52 @@ +// Copyright 2024 Antrea Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cluster + +import ( + "context" + "fmt" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type checkControlPlaneAvailability struct{} + +func init() { + RegisterTest("Check Control Plane Availability", &checkControlPlaneAvailability{}) +} + +func (t *checkControlPlaneAvailability) Run(ctx context.Context, testContext *testContext) error { + controlPlaneLabel := "node-role.kubernetes.io/control-plane" + masterNodeLabel := "node-role.kubernetes.io/master" + controlPlaneNode, err := testContext.client.CoreV1().Nodes().List(ctx, metav1.ListOptions{LabelSelector: controlPlaneLabel}) + if err != nil { + return fmt.Errorf("failed to list control plane Nodes: %w", err) + } + masterNode, err := testContext.client.CoreV1().Nodes().List(ctx, metav1.ListOptions{LabelSelector: masterNodeLabel}) + if err != nil { + return fmt.Errorf("failed to list master Nodes: %w", err) + } + if len(controlPlaneNode.Items) == 0 && len(masterNode.Items) == 0 { + testContext.Log("No control-plane Nodes were found; if installing Antrea in encap mode, some K8s functionalities (API aggregation, apiserver proxy, admission controllers) may be impacted.") + } else { + for _, node := range controlPlaneNode.Items { + testContext.Log("Control plane Node %s found", node.Name) + } + for _, node := range masterNode.Items { + testContext.Log("Master Node %s found", node.Name) + } + } + return nil +} diff --git a/pkg/antctl/raw/check/cluster/test_checkk8sversion.go b/pkg/antctl/raw/check/cluster/test_checkk8sversion.go new file mode 100644 index 00000000000..81dc1ff824c --- /dev/null +++ b/pkg/antctl/raw/check/cluster/test_checkk8sversion.go @@ -0,0 +1,48 @@ +// Copyright 2024 Antrea Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cluster + +import ( + "context" + "fmt" + "strings" + + "github.com/blang/semver" +) + +type checkK8sVersion struct{} + +func init() { + RegisterTest("Check K8s Version", &checkK8sVersion{}) +} + +func (t *checkK8sVersion) Run(ctx context.Context, testContext *testContext) error { + discoveryClient := testContext.client.Discovery() + serverVersion, err := discoveryClient.ServerVersion() + if err != nil { + return fmt.Errorf("error getting server version: %v", err) + } + currentVersion, err := semver.Parse(strings.TrimPrefix(serverVersion.GitVersion, "v")) + if err != nil { + return fmt.Errorf("error parsing server version: %v", err) + } + minVersion, _ := semver.Parse("1.19") + if currentVersion.GTE(minVersion) { + testContext.Log("Kubernetes server version is compatible with Antrea. Kubernetes version: %s", serverVersion.GitVersion) + } else { + testContext.Log("Kubernetes min version required: 1.19") + } + return nil +} diff --git a/pkg/antctl/raw/check/cluster/test_checkovsloadable.go b/pkg/antctl/raw/check/cluster/test_checkovsloadable.go new file mode 100644 index 00000000000..a7ce008fd0c --- /dev/null +++ b/pkg/antctl/raw/check/cluster/test_checkovsloadable.go @@ -0,0 +1,64 @@ +// Copyright 2024 Antrea Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cluster + +import ( + "context" + "fmt" + "strings" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "antrea.io/antrea/pkg/antctl/raw/check" +) + +type checkOVSLoadable struct{} + +func init() { + RegisterTest("Check if Openvswitch is Loadable", &checkOVSLoadable{}) +} + +func (c *checkOVSLoadable) Run(ctx context.Context, testContext *testContext) error { + pods, err := testContext.client.CoreV1().Pods(antreaNamespace).List(ctx, metav1.ListOptions{LabelSelector: "name=cluster-check"}) + if err != nil { + return fmt.Errorf("failed to list Pods: %v", err) + } + command := []string{ + "/bin/sh", + "-c", + `path="/lib/modules/$(uname -r)/modules.builtin"; grep -q "openvswitch.ko" "$path"; echo $?`, + } + stdout, _, err := check.ExecInPod(ctx, testContext.client, testContext.config, antreaNamespace, pods.Items[0].Name, "", command) + if err != nil { + return fmt.Errorf("error executing command in Pod %s: %v", pods.Items[0].Name, err) + } + if strings.TrimSpace(stdout) == "0" { + testContext.Log("Open vSwitch kernel module is built-in") + } else if strings.TrimSpace(stdout) == "1" { + testContext.Log("Open vSwitch kernel module is not built-in. Running modprobe command to load the module") + cmd := []string{"modprobe", "openvswitch"} + stdout, stderr, err := check.ExecInPod(ctx, testContext.client, testContext.config, antreaNamespace, pods.Items[0].Name, "", cmd) + if err != nil { + return fmt.Errorf("error executing modprobe command in Pod %s: %v", pods.Items[0].Name, err) + } + if stderr != "" { + testContext.Log("failed to load the OVS kernel module from the container, try running 'modprobe openvswitch' on your Nodes") + } + if stdout == "" { + testContext.Log("Open vSwitch kernel module loaded successfully") + } + } + return nil +} diff --git a/pkg/antctl/raw/check/installation/command.go b/pkg/antctl/raw/check/installation/command.go index 405f7b13f04..adfa3bc8d7f 100644 --- a/pkg/antctl/raw/check/installation/command.go +++ b/pkg/antctl/raw/check/installation/command.go @@ -23,7 +23,6 @@ import ( "time" "github.com/spf13/cobra" - appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" @@ -132,68 +131,6 @@ func newService(name string, selector map[string]string, port int) *corev1.Servi } } -type deploymentParameters struct { - Name string - Role string - Image string - Replicas int - Port int - Command []string - Affinity *corev1.Affinity - Tolerations []corev1.Toleration - Labels map[string]string -} - -func newDeployment(p deploymentParameters) *appsv1.Deployment { - if p.Replicas == 0 { - p.Replicas = 1 - } - replicas32 := int32(p.Replicas) - labels := map[string]string{ - "name": p.Name, - "kind": p.Role, - } - return &appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{ - Name: p.Name, - Labels: labels, - }, - Spec: appsv1.DeploymentSpec{ - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Name: p.Name, - Labels: labels, - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: p.Name, - Env: []corev1.EnvVar{ - {Name: "PORT", Value: fmt.Sprintf("%d", p.Port)}, - }, - Ports: []corev1.ContainerPort{ - {ContainerPort: int32(p.Port)}, - }, - Image: p.Image, - ImagePullPolicy: corev1.PullIfNotPresent, - Command: p.Command, - }, - }, - Affinity: p.Affinity, - Tolerations: p.Tolerations, - }, - }, - Replicas: &replicas32, - Selector: &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "name": p.Name, - "kind": p.Role, - }, - }, - }, - } -} - func NewTestContext(client kubernetes.Interface, config *rest.Config, clusterName string, o *options) *testContext { return &testContext{ client: client, @@ -259,7 +196,7 @@ func (t *testContext) setup(ctx context.Context) error { Effect: "NoSchedule", }, } - echoDeployment := newDeployment(deploymentParameters{ + echoDeployment := check.NewDeployment(check.DeploymentParameters{ Name: echoSameNodeDeploymentName, Role: kindEchoName, Port: 80, @@ -291,7 +228,7 @@ func (t *testContext) setup(ctx context.Context) error { return fmt.Errorf("unable to create Deployment %s: %s", echoSameNodeDeploymentName, err) } t.Log("Deploying client Deployment %s...", clientDeploymentName) - clientDeployment := newDeployment(deploymentParameters{ + clientDeployment := check.NewDeployment(check.DeploymentParameters{ Name: clientDeploymentName, Role: kindClientName, Image: deploymentImage, @@ -311,7 +248,7 @@ func (t *testContext) setup(ctx context.Context) error { if err != nil { return err } - echoOtherNodeDeployment := newDeployment(deploymentParameters{ + echoOtherNodeDeployment := check.NewDeployment(check.DeploymentParameters{ Name: echoOtherNodeDeploymentName, Role: kindEchoName, Port: 80, @@ -343,7 +280,7 @@ func (t *testContext) setup(ctx context.Context) error { if err != nil { return fmt.Errorf("unable to create Deployment %s: %s", echoOtherNodeDeploymentName, err) } - if err := t.waitForDeploymentsReady(ctx, time.Second, podReadyTimeout, clientDeploymentName, echoSameNodeDeploymentName, echoOtherNodeDeploymentName); err != nil { + if err := check.WaitForDeploymentsReady(ctx, time.Second, podReadyTimeout, t.client, t.namespace, t.clusterName, clientDeploymentName, echoSameNodeDeploymentName, echoOtherNodeDeploymentName); err != nil { return err } podList, err := t.client.CoreV1().Pods(t.namespace).List(ctx, metav1.ListOptions{LabelSelector: "name=" + echoOtherNodeDeploymentName}) @@ -355,7 +292,7 @@ func (t *testContext) setup(ctx context.Context) error { } } else { t.Log("skipping other Node Deployments as multiple Nodes are not available") - if err := t.waitForDeploymentsReady(ctx, time.Second, podReadyTimeout, clientDeploymentName, echoSameNodeDeploymentName); err != nil { + if err := check.WaitForDeploymentsReady(ctx, time.Second, podReadyTimeout, t.client, t.namespace, t.clusterName, clientDeploymentName, echoSameNodeDeploymentName); err != nil { return err } } @@ -375,24 +312,6 @@ func (t *testContext) setup(ctx context.Context) error { return nil } -func (t *testContext) waitForDeploymentsReady(ctx context.Context, interval, timeout time.Duration, deployments ...string) error { - for _, deployment := range deployments { - t.Log("Waiting for Deployment %s to become ready...", deployment) - err := wait.PollUntilContextTimeout(ctx, interval, timeout, false, func(ctx context.Context) (bool, error) { - ready, err := check.DeploymentIsReady(ctx, t.client, t.namespace, deployment) - if err != nil { - return false, fmt.Errorf("error checking readiness of Deployment %s: %w", deployment, err) - } - return ready, nil - }) - if err != nil { - return fmt.Errorf("waiting for Deployment %s to become ready has been interrupted: %w", deployment, err) - } - t.Log("Deployment %s is ready.", deployment) - } - return nil -} - func (t *testContext) Log(format string, a ...interface{}) { fmt.Fprintf(os.Stdout, fmt.Sprintf("[%s] ", t.clusterName)+format+"\n", a...) } diff --git a/pkg/antctl/raw/check/util.go b/pkg/antctl/raw/check/util.go index 2a6e29936e6..9696c07988a 100644 --- a/pkg/antctl/raw/check/util.go +++ b/pkg/antctl/raw/check/util.go @@ -18,10 +18,13 @@ import ( "bytes" "context" "fmt" + "os" + "time" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" @@ -103,3 +106,89 @@ func ExecInPod(ctx context.Context, client kubernetes.Interface, config *rest.Co } return stdout.String(), stderr.String(), nil } + +func NewDeployment(p DeploymentParameters) *appsv1.Deployment { + if p.Replicas == 0 { + p.Replicas = 1 + } + replicas32 := int32(p.Replicas) + labels := map[string]string{ + "name": p.Name, + "kind": p.Role, + } + var ports []corev1.ContainerPort + if p.Port > 0 { + ports = append(ports, corev1.ContainerPort{ContainerPort: int32(p.Port)}) + } + var env []corev1.EnvVar + if p.Port > 0 { + env = append(env, corev1.EnvVar{Name: "PORT", Value: fmt.Sprintf("%d", p.Port)}) + } + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: p.Name, + Labels: labels, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: &replicas32, + Selector: &metav1.LabelSelector{ + MatchLabels: labels, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labels, + }, + Spec: corev1.PodSpec{ + HostNetwork: p.HostNetwork, + Containers: []corev1.Container{ + { + Name: p.Name, + Image: p.Image, + Ports: ports, + Env: env, + ImagePullPolicy: corev1.PullIfNotPresent, + Command: p.Command, + VolumeMounts: p.VolumeMounts, + }, + }, + Tolerations: p.Tolerations, + Volumes: p.Volumes, + Affinity: p.Affinity, + }, + }, + }, + } +} + +type DeploymentParameters struct { + Name string + Role string + Image string + Replicas int + Port int + Command []string + Affinity *corev1.Affinity + Tolerations []corev1.Toleration + Labels map[string]string + VolumeMounts []corev1.VolumeMount + Volumes []corev1.Volume + HostNetwork bool +} + +func WaitForDeploymentsReady(ctx context.Context, interval, timeout time.Duration, client kubernetes.Interface, antreaNamespace string, clusterName string, deployments ...string) error { + for _, deployment := range deployments { + fmt.Fprintf(os.Stdout, fmt.Sprintf("[%s] ", clusterName)+"Waiting for Deployment %s to become ready..."+"\n", deployment) + err := wait.PollUntilContextTimeout(ctx, interval, timeout, false, func(ctx context.Context) (bool, error) { + ready, err := DeploymentIsReady(ctx, client, antreaNamespace, deployment) + if err != nil { + return false, fmt.Errorf("error checking readiness of Deployment %s: %w", deployment, err) + } + return ready, nil + }) + if err != nil { + return fmt.Errorf("waiting for Deployment %s to become ready has been interrupted: %w", deployment, err) + } + fmt.Fprintf(os.Stdout, fmt.Sprintf("[%s] ", clusterName)+"Deployment %s is ready."+"\n", deployment) + } + return nil +} From e75118fc345dd656217c0148e3b795254d3e91e0 Mon Sep 17 00:00:00 2001 From: Kanha gupta Date: Tue, 7 May 2024 00:37:39 +0530 Subject: [PATCH 02/12] Review 1 changes Signed-off-by: Kanha gupta --- .github/workflows/kind.yml | 6 +-- pkg/antctl/raw/check/cluster/command.go | 44 ++++++------------- ...Existence.go => test_checkcniexistence.go} | 27 ++++++------ .../test_checkcontrolplaneavailability.go | 4 +- .../raw/check/cluster/test_checkk8sversion.go | 2 +- .../check/cluster/test_checkovsloadable.go | 14 +++--- pkg/antctl/raw/check/installation/command.go | 37 +--------------- pkg/antctl/raw/check/util.go | 43 ++++++++++++++++-- 8 files changed, 82 insertions(+), 95 deletions(-) rename pkg/antctl/raw/check/cluster/{test_checkCNIExistence.go => test_checkcniexistence.go} (56%) diff --git a/.github/workflows/kind.yml b/.github/workflows/kind.yml index 08bd2934b81..a5c3278ec2b 100644 --- a/.github/workflows/kind.yml +++ b/.github/workflows/kind.yml @@ -741,7 +741,7 @@ jobs: path: log.tar.gz retention-days: 30 - run-post-installation-checks: + run-installation-checks: name: Test connectivity using 'antctl check' command needs: [ build-antrea-coverage-image ] runs-on: [ ubuntu-latest ] @@ -775,14 +775,14 @@ jobs: - name: Build antctl binary run: | make antctl-linux - - name: Run Pre-installation checks + - name: Run Pre checks run: | ./bin/antctl-linux check cluster - name: Load Docker images and deploy Antrea run: | kind load docker-image antrea/antrea-controller-ubuntu-coverage:latest antrea/antrea-agent-ubuntu-coverage:latest kubectl apply -f build/yamls/antrea.yml - - name: Run antctl command + - name: Run Post checks run: | ./bin/antctl-linux check installation diff --git a/pkg/antctl/raw/check/cluster/command.go b/pkg/antctl/raw/check/cluster/command.go index b8c5a3582fc..cbdb4767d9d 100644 --- a/pkg/antctl/raw/check/cluster/command.go +++ b/pkg/antctl/raw/check/cluster/command.go @@ -21,11 +21,8 @@ import ( "time" "github.com/spf13/cobra" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/utils/ptr" @@ -58,7 +55,8 @@ func newOptions() *options { const ( antreaNamespace = "kube-system" - deploymentName = "cluster-check" + testNamespace = "antrea-test" + deploymentName = "check-cluster" podReadyTimeout = 1 * time.Minute ) @@ -77,6 +75,7 @@ type testContext struct { config *rest.Config clusterName string antreaNamespace string + namespace string } func Run(o *options) error { @@ -98,17 +97,22 @@ func Run(o *options) error { } } testContext.Log("Test finished") - testContext.teardown(ctx, deploymentName, antreaNamespace) + check.Teardown(ctx, testContext.client, testContext.namespace, testContext.clusterName) return nil } func (t *testContext) setup(ctx context.Context) error { + t.Log("Creating Namespace %s for pre installation tests...", t.namespace) + _, err := t.client.CoreV1().Namespaces().Create(ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: t.namespace}}, metav1.CreateOptions{}) + if err != nil { + return fmt.Errorf("unable to create Namespace %s: %s", t.namespace, err) + } deployment := check.NewDeployment(check.DeploymentParameters{ Name: deploymentName, Image: "antrea/antrea-agent-ubuntu:latest", Replicas: 1, Command: []string{"sleep", "infinity"}, - Labels: map[string]string{"app": "cluster-check"}, + Labels: map[string]string{"app": "check-cluster"}, HostNetwork: true, VolumeMounts: []corev1.VolumeMount{ {Name: "cni-conf", MountPath: "/etc/cni/net.d"}, @@ -153,13 +157,13 @@ func (t *testContext) setup(ctx context.Context) error { }) t.Log("Creating Deployment") - _, err := t.client.AppsV1().Deployments(antreaNamespace).Create(ctx, deployment, metav1.CreateOptions{}) + _, err = t.client.AppsV1().Deployments(t.namespace).Create(ctx, deployment, metav1.CreateOptions{}) if err != nil { return fmt.Errorf("unable to create Deployment: %w", err) } t.Log("Waiting for Deployment to become ready") - check.WaitForDeploymentsReady(ctx, time.Second, podReadyTimeout, t.client, t.antreaNamespace, t.clusterName, deploymentName) + check.WaitForDeploymentsReady(ctx, time.Second, podReadyTimeout, t.client, t.namespace, t.clusterName, deploymentName) if err != nil { return fmt.Errorf("error while waiting for Deployment to become ready: %w", err) } @@ -172,32 +176,10 @@ func NewTestContext(client kubernetes.Interface, config *rest.Config, clusterNam config: config, clusterName: clusterName, antreaNamespace: o.antreaNamespace, + namespace: check.GenerateRandomNamespace(testNamespace), } } -func (t *testContext) teardown(ctx context.Context, deploymentName, namespace string) error { - err := t.client.AppsV1().Deployments(namespace).Delete(ctx, deploymentName, metav1.DeleteOptions{}) - if err != nil { - return err - } - t.Log("Waiting for the deletion of Deployment %s in Namespace %s...", deploymentName, namespace) - err = wait.PollUntilContextTimeout(ctx, 2*time.Second, 1*time.Minute, true, func(ctx context.Context) (bool, error) { - _, err := t.client.AppsV1().Deployments(namespace).Get(ctx, deploymentName, metav1.GetOptions{}) - if errors.IsNotFound(err) { - return true, nil - } - if err != nil { - return false, err - } - return false, nil - }) - if err != nil { - return fmt.Errorf("error waiting for Deployment %s to be deleted in Namespace %s: %w", deploymentName, namespace, err) - } - t.Log("Deployment %s successfully deleted from Namespace %s", deploymentName, namespace) - return nil -} - func (t *testContext) Log(format string, a ...interface{}) { fmt.Fprintf(os.Stdout, fmt.Sprintf("[%s] ", t.clusterName)+format+"\n", a...) } diff --git a/pkg/antctl/raw/check/cluster/test_checkCNIExistence.go b/pkg/antctl/raw/check/cluster/test_checkcniexistence.go similarity index 56% rename from pkg/antctl/raw/check/cluster/test_checkCNIExistence.go rename to pkg/antctl/raw/check/cluster/test_checkcniexistence.go index 6fd7b8e8799..ee02feb2de6 100644 --- a/pkg/antctl/raw/check/cluster/test_checkCNIExistence.go +++ b/pkg/antctl/raw/check/cluster/test_checkcniexistence.go @@ -28,32 +28,33 @@ import ( type checkCNIExistence struct{} func init() { - RegisterTest("Check if another CNI is Present", &checkCNIExistence{}) + RegisterTest("Check if another CNI is present", &checkCNIExistence{}) } func (t *checkCNIExistence) Run(ctx context.Context, testContext *testContext) error { - pods, err := testContext.client.CoreV1().Pods(antreaNamespace).List(ctx, metav1.ListOptions{LabelSelector: "name=cluster-check"}) + pods, err := testContext.client.CoreV1().Pods(testContext.namespace).List(ctx, metav1.ListOptions{LabelSelector: "name=check-cluster"}) if err != nil { return fmt.Errorf("failed to list Pods: %v", err) } - testContext.Log("Checking CNI configurations in Pod: %s", pods.Items[0].Name) - command := []string{"ls", "/etc/cni/net.d"} - output, _, err := check.ExecInPod(ctx, testContext.client, testContext.config, antreaNamespace, pods.Items[0].Name, "", command) + command := []string{"ls", "-1", "/etc/cni/net.d"} + output, _, err := check.ExecInPod(ctx, testContext.client, testContext.config, testContext.namespace, pods.Items[0].Name, "", command) if err != nil { - testContext.Log("Failed to execute command in pod: %s, error: %v", pods.Items[0].Name, err) + testContext.Log("Failed to execute command in Pod: %s, error: %v", pods.Items[0].Name, err) } outputStr := strings.TrimSpace(output) if outputStr == "" { - testContext.Log("No files present in /etc/cni/net.d in pod: %s", pods.Items[0].Name) + testContext.Log("No files present in /etc/cni/net.d in Pod: %s", pods.Items[0].Name) } else { files := strings.Split(outputStr, "\n") sort.Strings(files) - if len(files) > 0 && files[0] < "10-antrea.conflist" { - testContext.Log("Warning: Another CNI configuration file with higher priority than Antrea's CNI configuration file found: %s", files[0]) - } else if len(files) > 0 && files[0] != "10-antrea.conflist" { - testContext.Log("Warning: Another CNI configuration file found: %s.", files[0]) - } else { - testContext.Log("Antrea's CNI configuration file already present: %s", files[0]) + if len(files) > 0 { + if files[0] < "10-antrea.conflist" { + testContext.Log("Warning: Another CNI configuration file with higher priority than Antrea's CNI configuration file found: %s", files[0]) + } else if files[0] != "10-antrea.conflist" { + testContext.Log("Warning: Another CNI configuration file found: %s", files[0]) + } else { + testContext.Log("Antrea's CNI configuration file already present: %s", files[0]) + } } } return nil diff --git a/pkg/antctl/raw/check/cluster/test_checkcontrolplaneavailability.go b/pkg/antctl/raw/check/cluster/test_checkcontrolplaneavailability.go index 3b2f29851b1..25d56c4baae 100644 --- a/pkg/antctl/raw/check/cluster/test_checkcontrolplaneavailability.go +++ b/pkg/antctl/raw/check/cluster/test_checkcontrolplaneavailability.go @@ -24,7 +24,7 @@ import ( type checkControlPlaneAvailability struct{} func init() { - RegisterTest("Check Control Plane Availability", &checkControlPlaneAvailability{}) + RegisterTest("Check control plane Nodes availability", &checkControlPlaneAvailability{}) } func (t *checkControlPlaneAvailability) Run(ctx context.Context, testContext *testContext) error { @@ -39,7 +39,7 @@ func (t *checkControlPlaneAvailability) Run(ctx context.Context, testContext *te return fmt.Errorf("failed to list master Nodes: %w", err) } if len(controlPlaneNode.Items) == 0 && len(masterNode.Items) == 0 { - testContext.Log("No control-plane Nodes were found; if installing Antrea in encap mode, some K8s functionalities (API aggregation, apiserver proxy, admission controllers) may be impacted.") + testContext.Log("No control-plane or master Nodes were found; if installing Antrea in encap mode, some K8s functionalities (API aggregation, apiserver proxy, admission controllers) may be impacted.") } else { for _, node := range controlPlaneNode.Items { testContext.Log("Control plane Node %s found", node.Name) diff --git a/pkg/antctl/raw/check/cluster/test_checkk8sversion.go b/pkg/antctl/raw/check/cluster/test_checkk8sversion.go index 81dc1ff824c..057e2d6c280 100644 --- a/pkg/antctl/raw/check/cluster/test_checkk8sversion.go +++ b/pkg/antctl/raw/check/cluster/test_checkk8sversion.go @@ -25,7 +25,7 @@ import ( type checkK8sVersion struct{} func init() { - RegisterTest("Check K8s Version", &checkK8sVersion{}) + RegisterTest("Check K8s version", &checkK8sVersion{}) } func (t *checkK8sVersion) Run(ctx context.Context, testContext *testContext) error { diff --git a/pkg/antctl/raw/check/cluster/test_checkovsloadable.go b/pkg/antctl/raw/check/cluster/test_checkovsloadable.go index a7ce008fd0c..fcbe1d3449f 100644 --- a/pkg/antctl/raw/check/cluster/test_checkovsloadable.go +++ b/pkg/antctl/raw/check/cluster/test_checkovsloadable.go @@ -27,11 +27,11 @@ import ( type checkOVSLoadable struct{} func init() { - RegisterTest("Check if Openvswitch is Loadable", &checkOVSLoadable{}) + RegisterTest("Check if the module openvswitch is loadable", &checkOVSLoadable{}) } func (c *checkOVSLoadable) Run(ctx context.Context, testContext *testContext) error { - pods, err := testContext.client.CoreV1().Pods(antreaNamespace).List(ctx, metav1.ListOptions{LabelSelector: "name=cluster-check"}) + pods, err := testContext.client.CoreV1().Pods(testContext.namespace).List(ctx, metav1.ListOptions{LabelSelector: "name=check-cluster"}) if err != nil { return fmt.Errorf("failed to list Pods: %v", err) } @@ -40,16 +40,16 @@ func (c *checkOVSLoadable) Run(ctx context.Context, testContext *testContext) er "-c", `path="/lib/modules/$(uname -r)/modules.builtin"; grep -q "openvswitch.ko" "$path"; echo $?`, } - stdout, _, err := check.ExecInPod(ctx, testContext.client, testContext.config, antreaNamespace, pods.Items[0].Name, "", command) + stdout, _, err := check.ExecInPod(ctx, testContext.client, testContext.config, testContext.namespace, pods.Items[0].Name, "", command) if err != nil { return fmt.Errorf("error executing command in Pod %s: %v", pods.Items[0].Name, err) } if strings.TrimSpace(stdout) == "0" { - testContext.Log("Open vSwitch kernel module is built-in") + testContext.Log("The kernel module openvswitch is built-in") } else if strings.TrimSpace(stdout) == "1" { - testContext.Log("Open vSwitch kernel module is not built-in. Running modprobe command to load the module") + testContext.Log("The kernel module openvswitch is not built-in. Running modprobe command to load the module.") cmd := []string{"modprobe", "openvswitch"} - stdout, stderr, err := check.ExecInPod(ctx, testContext.client, testContext.config, antreaNamespace, pods.Items[0].Name, "", cmd) + stdout, stderr, err := check.ExecInPod(ctx, testContext.client, testContext.config, testContext.namespace, pods.Items[0].Name, "", cmd) if err != nil { return fmt.Errorf("error executing modprobe command in Pod %s: %v", pods.Items[0].Name, err) } @@ -57,7 +57,7 @@ func (c *checkOVSLoadable) Run(ctx context.Context, testContext *testContext) er testContext.Log("failed to load the OVS kernel module from the container, try running 'modprobe openvswitch' on your Nodes") } if stdout == "" { - testContext.Log("Open vSwitch kernel module loaded successfully") + testContext.Log("openvswitch kernel module loaded successfully") } } return nil diff --git a/pkg/antctl/raw/check/installation/command.go b/pkg/antctl/raw/check/installation/command.go index adfa3bc8d7f..5f0a4668a25 100644 --- a/pkg/antctl/raw/check/installation/command.go +++ b/pkg/antctl/raw/check/installation/command.go @@ -16,7 +16,6 @@ package installation import ( "context" - "crypto/rand" "fmt" "net" "os" @@ -25,7 +24,6 @@ import ( "github.com/spf13/cobra" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" @@ -107,7 +105,7 @@ func Run(o *options) error { } } testContext.Log("Test finished") - testContext.teardown(ctx) + check.Teardown(ctx, testContext.client, testContext.namespace, testContext.clusterName) return nil } @@ -137,38 +135,7 @@ func NewTestContext(client kubernetes.Interface, config *rest.Config, clusterNam config: config, clusterName: clusterName, antreaNamespace: o.antreaNamespace, - namespace: generateRandomNamespace(testNamespacePrefix), - } -} - -func generateRandomNamespace(baseName string) string { - const letters = "abcdefghijklmnopqrstuvwxyz0123456789" - bytes := make([]byte, 5) - _, err := rand.Read(bytes) - if err != nil { - panic(err) - } - for i, b := range bytes { - bytes[i] = letters[b%byte(len(letters))] - } - return fmt.Sprintf("%s-%s", baseName, string(bytes)) -} - -func (t *testContext) teardown(ctx context.Context) { - t.Log("Deleting post installation tests setup...") - t.client.CoreV1().Namespaces().Delete(ctx, t.namespace, metav1.DeleteOptions{}) - t.Log("Waiting for Namespace %s to disappear", t.namespace) - err := wait.PollUntilContextTimeout(ctx, 2*time.Second, 1*time.Minute, true, func(ctx context.Context) (bool, error) { - _, err := t.client.CoreV1().Namespaces().Get(ctx, t.namespace, metav1.GetOptions{}) - if err != nil { - return true, nil - } - return false, nil - }) - if err != nil { - t.Log("Setup deletion failed") - } else { - t.Log("Setup deletion successful") + namespace: check.GenerateRandomNamespace(testNamespacePrefix), } } diff --git a/pkg/antctl/raw/check/util.go b/pkg/antctl/raw/check/util.go index 9696c07988a..f5699204e72 100644 --- a/pkg/antctl/raw/check/util.go +++ b/pkg/antctl/raw/check/util.go @@ -17,6 +17,7 @@ package check import ( "bytes" "context" + "crypto/rand" "fmt" "os" "time" @@ -175,9 +176,14 @@ type DeploymentParameters struct { HostNetwork bool } -func WaitForDeploymentsReady(ctx context.Context, interval, timeout time.Duration, client kubernetes.Interface, antreaNamespace string, clusterName string, deployments ...string) error { +func WaitForDeploymentsReady(ctx context.Context, + interval, timeout time.Duration, + client kubernetes.Interface, + antreaNamespace string, + clusterName string, + deployments ...string) error { for _, deployment := range deployments { - fmt.Fprintf(os.Stdout, fmt.Sprintf("[%s] ", clusterName)+"Waiting for Deployment %s to become ready..."+"\n", deployment) + fmt.Fprintf(os.Stdout, fmt.Sprintf("[%s] ", clusterName)+"Waiting for Deployment %s to become ready...\n", deployment) err := wait.PollUntilContextTimeout(ctx, interval, timeout, false, func(ctx context.Context) (bool, error) { ready, err := DeploymentIsReady(ctx, client, antreaNamespace, deployment) if err != nil { @@ -188,7 +194,38 @@ func WaitForDeploymentsReady(ctx context.Context, interval, timeout time.Duratio if err != nil { return fmt.Errorf("waiting for Deployment %s to become ready has been interrupted: %w", deployment, err) } - fmt.Fprintf(os.Stdout, fmt.Sprintf("[%s] ", clusterName)+"Deployment %s is ready."+"\n", deployment) + fmt.Fprintf(os.Stdout, fmt.Sprintf("[%s] ", clusterName)+"Deployment %s is ready.\n", deployment) } return nil } + +func GenerateRandomNamespace(baseName string) string { + const letters = "abcdefghijklmnopqrstuvwxyz0123456789" + bytes := make([]byte, 5) + _, err := rand.Read(bytes) + if err != nil { + panic(err) + } + for i, b := range bytes { + bytes[i] = letters[b%byte(len(letters))] + } + return fmt.Sprintf("%s-%s", baseName, string(bytes)) +} + +func Teardown(ctx context.Context, client kubernetes.Interface, namespace string, clusterName string) { + fmt.Fprintf(os.Stdout, fmt.Sprintf("[%s] ", clusterName)+"Deleting post installation tests setup...\n") + client.CoreV1().Namespaces().Delete(ctx, namespace, metav1.DeleteOptions{}) + fmt.Fprintf(os.Stdout, fmt.Sprintf("[%s] ", clusterName)+"Waiting for Namespace %s to disappear \n", namespace) + err := wait.PollUntilContextTimeout(ctx, 2*time.Second, 1*time.Minute, true, func(ctx context.Context) (bool, error) { + _, err := client.CoreV1().Namespaces().Get(ctx, namespace, metav1.GetOptions{}) + if err != nil { + return true, nil + } + return false, nil + }) + if err != nil { + fmt.Fprintf(os.Stdout, fmt.Sprintf("[%s] ", clusterName)+"Setup deletion failed \n") + } else { + fmt.Fprintf(os.Stdout, fmt.Sprintf("[%s] ", clusterName)+"Setup deletion successful \n") + } +} From 93afe4041497f2accaa179f58beb87a4374f16e5 Mon Sep 17 00:00:00 2001 From: Kanha gupta Date: Tue, 7 May 2024 03:48:26 +0530 Subject: [PATCH 03/12] Review 2 changes Signed-off-by: Kanha gupta --- pkg/antctl/raw/check/cluster/command.go | 6 ++-- .../check/cluster/test_checkcniexistence.go | 4 +-- .../test_checkcontrolplaneavailability.go | 29 +++++++++---------- pkg/antctl/raw/check/installation/command.go | 6 ++-- pkg/antctl/raw/check/util.go | 6 ++-- 5 files changed, 25 insertions(+), 26 deletions(-) diff --git a/pkg/antctl/raw/check/cluster/command.go b/pkg/antctl/raw/check/cluster/command.go index cbdb4767d9d..e297f06f0ba 100644 --- a/pkg/antctl/raw/check/cluster/command.go +++ b/pkg/antctl/raw/check/cluster/command.go @@ -39,7 +39,7 @@ func Command() *cobra.Command { return Run(o) }, } - command.Flags().StringVarP(&o.antreaNamespace, "Namespace", "n", o.antreaNamespace, "Configure Namespace in which Antrea is running") + command.Flags().StringVarP(&o.antreaNamespace, "namespace", "n", o.antreaNamespace, "Configure Namespace in which Antrea is running") return command } @@ -97,7 +97,7 @@ func Run(o *options) error { } } testContext.Log("Test finished") - check.Teardown(ctx, testContext.client, testContext.namespace, testContext.clusterName) + check.Teardown(ctx, testContext.client, testContext.clusterName, testContext.namespace) return nil } @@ -163,7 +163,7 @@ func (t *testContext) setup(ctx context.Context) error { } t.Log("Waiting for Deployment to become ready") - check.WaitForDeploymentsReady(ctx, time.Second, podReadyTimeout, t.client, t.namespace, t.clusterName, deploymentName) + check.WaitForDeploymentsReady(ctx, time.Second, podReadyTimeout, t.client, t.clusterName, t.namespace, deploymentName) if err != nil { return fmt.Errorf("error while waiting for Deployment to become ready: %w", err) } diff --git a/pkg/antctl/raw/check/cluster/test_checkcniexistence.go b/pkg/antctl/raw/check/cluster/test_checkcniexistence.go index ee02feb2de6..7ab8acd619a 100644 --- a/pkg/antctl/raw/check/cluster/test_checkcniexistence.go +++ b/pkg/antctl/raw/check/cluster/test_checkcniexistence.go @@ -49,9 +49,9 @@ func (t *checkCNIExistence) Run(ctx context.Context, testContext *testContext) e sort.Strings(files) if len(files) > 0 { if files[0] < "10-antrea.conflist" { - testContext.Log("Warning: Another CNI configuration file with higher priority than Antrea's CNI configuration file found: %s", files[0]) + testContext.Log("Another CNI configuration file with higher priority than Antrea's CNI configuration file found: %s", files[0]) } else if files[0] != "10-antrea.conflist" { - testContext.Log("Warning: Another CNI configuration file found: %s", files[0]) + testContext.Log("Another CNI configuration file found: %s with Antrea having higher precedence", files[0]) } else { testContext.Log("Antrea's CNI configuration file already present: %s", files[0]) } diff --git a/pkg/antctl/raw/check/cluster/test_checkcontrolplaneavailability.go b/pkg/antctl/raw/check/cluster/test_checkcontrolplaneavailability.go index 25d56c4baae..22e00ba6213 100644 --- a/pkg/antctl/raw/check/cluster/test_checkcontrolplaneavailability.go +++ b/pkg/antctl/raw/check/cluster/test_checkcontrolplaneavailability.go @@ -19,6 +19,7 @@ import ( "fmt" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" ) type checkControlPlaneAvailability struct{} @@ -28,24 +29,22 @@ func init() { } func (t *checkControlPlaneAvailability) Run(ctx context.Context, testContext *testContext) error { - controlPlaneLabel := "node-role.kubernetes.io/control-plane" - masterNodeLabel := "node-role.kubernetes.io/master" - controlPlaneNode, err := testContext.client.CoreV1().Nodes().List(ctx, metav1.ListOptions{LabelSelector: controlPlaneLabel}) - if err != nil { - return fmt.Errorf("failed to list control plane Nodes: %w", err) - } - masterNode, err := testContext.client.CoreV1().Nodes().List(ctx, metav1.ListOptions{LabelSelector: masterNodeLabel}) - if err != nil { - return fmt.Errorf("failed to list master Nodes: %w", err) + controlPlaneNodes := sets.New[string]() + controlPlaneLabels := []string{"node-role.kubernetes.io/control-plane", "node-role.kubernetes.io/master"} + for _, label := range controlPlaneLabels { + nodes, err := testContext.client.CoreV1().Nodes().List(ctx, metav1.ListOptions{LabelSelector: label}) + if err != nil { + return fmt.Errorf("failed to list Nodes with label %s: %w", label, err) + } + for idx := range nodes.Items { + controlPlaneNodes.Insert(nodes.Items[idx].Name) + } } - if len(controlPlaneNode.Items) == 0 && len(masterNode.Items) == 0 { + if controlPlaneNodes.Len() == 0 { testContext.Log("No control-plane or master Nodes were found; if installing Antrea in encap mode, some K8s functionalities (API aggregation, apiserver proxy, admission controllers) may be impacted.") } else { - for _, node := range controlPlaneNode.Items { - testContext.Log("Control plane Node %s found", node.Name) - } - for _, node := range masterNode.Items { - testContext.Log("Master Node %s found", node.Name) + for _, node := range controlPlaneNodes.UnsortedList() { + testContext.Log("Nodes found : %s", node) } } return nil diff --git a/pkg/antctl/raw/check/installation/command.go b/pkg/antctl/raw/check/installation/command.go index 5f0a4668a25..bd09ba4685c 100644 --- a/pkg/antctl/raw/check/installation/command.go +++ b/pkg/antctl/raw/check/installation/command.go @@ -105,7 +105,7 @@ func Run(o *options) error { } } testContext.Log("Test finished") - check.Teardown(ctx, testContext.client, testContext.namespace, testContext.clusterName) + check.Teardown(ctx, testContext.client, testContext.clusterName, testContext.namespace) return nil } @@ -247,7 +247,7 @@ func (t *testContext) setup(ctx context.Context) error { if err != nil { return fmt.Errorf("unable to create Deployment %s: %s", echoOtherNodeDeploymentName, err) } - if err := check.WaitForDeploymentsReady(ctx, time.Second, podReadyTimeout, t.client, t.namespace, t.clusterName, clientDeploymentName, echoSameNodeDeploymentName, echoOtherNodeDeploymentName); err != nil { + if err := check.WaitForDeploymentsReady(ctx, time.Second, podReadyTimeout, t.client, t.clusterName, t.namespace, clientDeploymentName, echoSameNodeDeploymentName, echoOtherNodeDeploymentName); err != nil { return err } podList, err := t.client.CoreV1().Pods(t.namespace).List(ctx, metav1.ListOptions{LabelSelector: "name=" + echoOtherNodeDeploymentName}) @@ -259,7 +259,7 @@ func (t *testContext) setup(ctx context.Context) error { } } else { t.Log("skipping other Node Deployments as multiple Nodes are not available") - if err := check.WaitForDeploymentsReady(ctx, time.Second, podReadyTimeout, t.client, t.namespace, t.clusterName, clientDeploymentName, echoSameNodeDeploymentName); err != nil { + if err := check.WaitForDeploymentsReady(ctx, time.Second, podReadyTimeout, t.client, t.clusterName, t.namespace, clientDeploymentName, echoSameNodeDeploymentName); err != nil { return err } } diff --git a/pkg/antctl/raw/check/util.go b/pkg/antctl/raw/check/util.go index f5699204e72..429702b650c 100644 --- a/pkg/antctl/raw/check/util.go +++ b/pkg/antctl/raw/check/util.go @@ -179,13 +179,13 @@ type DeploymentParameters struct { func WaitForDeploymentsReady(ctx context.Context, interval, timeout time.Duration, client kubernetes.Interface, - antreaNamespace string, clusterName string, + namespace string, deployments ...string) error { for _, deployment := range deployments { fmt.Fprintf(os.Stdout, fmt.Sprintf("[%s] ", clusterName)+"Waiting for Deployment %s to become ready...\n", deployment) err := wait.PollUntilContextTimeout(ctx, interval, timeout, false, func(ctx context.Context) (bool, error) { - ready, err := DeploymentIsReady(ctx, client, antreaNamespace, deployment) + ready, err := DeploymentIsReady(ctx, client, namespace, deployment) if err != nil { return false, fmt.Errorf("error checking readiness of Deployment %s: %w", deployment, err) } @@ -212,7 +212,7 @@ func GenerateRandomNamespace(baseName string) string { return fmt.Sprintf("%s-%s", baseName, string(bytes)) } -func Teardown(ctx context.Context, client kubernetes.Interface, namespace string, clusterName string) { +func Teardown(ctx context.Context, client kubernetes.Interface, clusterName string, namespace string) { fmt.Fprintf(os.Stdout, fmt.Sprintf("[%s] ", clusterName)+"Deleting post installation tests setup...\n") client.CoreV1().Namespaces().Delete(ctx, namespace, metav1.DeleteOptions{}) fmt.Fprintf(os.Stdout, fmt.Sprintf("[%s] ", clusterName)+"Waiting for Namespace %s to disappear \n", namespace) From 1cf55f9a585b32b988c916be08c5177519adf41a Mon Sep 17 00:00:00 2001 From: Kanha gupta Date: Wed, 8 May 2024 01:49:02 +0530 Subject: [PATCH 04/12] Improve readability of results Signed-off-by: Kanha gupta --- .github/workflows/kind.yml | 15 +++++++++------ pkg/antctl/raw/check/cluster/command.go | 25 ++++++++++++++++++++++--- 2 files changed, 31 insertions(+), 9 deletions(-) diff --git a/.github/workflows/kind.yml b/.github/workflows/kind.yml index 253bcceebaf..bd01f15d644 100644 --- a/.github/workflows/kind.yml +++ b/.github/workflows/kind.yml @@ -741,8 +741,8 @@ jobs: path: log.tar.gz retention-days: 30 - run-post-installation-checks: - name: Test connectivity using 'antctl check' command + run-installation-checks: + name: Test installation using 'antctl check' command needs: [ build-antrea-coverage-image ] runs-on: [ ubuntu-latest ] steps: @@ -772,13 +772,16 @@ jobs: - name: Create Kind Cluster run: | ./ci/kind/kind-setup.sh create kind --ip-family dual - - name: Deploy Antrea - run: | - kubectl apply -f build/yamls/antrea.yml - name: Build antctl binary run: | make antctl-linux - - name: Run antctl command + - name: Run Pre checks + run: | + /bin/antctl-linux check cluster + - name: Deploy Antrea + run: | + kubectl apply -f build/yamls/antrea.yml + - name: Run Post checks run: | ./bin/antctl-linux check installation diff --git a/pkg/antctl/raw/check/cluster/command.go b/pkg/antctl/raw/check/cluster/command.go index e297f06f0ba..a18a89cef09 100644 --- a/pkg/antctl/raw/check/cluster/command.go +++ b/pkg/antctl/raw/check/cluster/command.go @@ -20,6 +20,7 @@ import ( "os" "time" + "github.com/fatih/color" "github.com/spf13/cobra" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -88,16 +89,22 @@ func Run(o *options) error { if err := testContext.setup(ctx); err != nil { return err } + var numSuccess, numFailure int for name, test := range testsRegistry { testContext.Header("Running test: %s", name) if err := test.Run(ctx, testContext); err != nil { - testContext.Header("Test %s failed: %s", name, err) + testContext.Fail("Test %s failed: %v", name, err) + numFailure++ } else { - testContext.Header("Test %s passed", name) + testContext.Success("Test %s passed", name) + numSuccess++ } } - testContext.Log("Test finished") + testContext.Log("Test finished: %v tests succeeded, %v tests failed ", numSuccess, numFailure) check.Teardown(ctx, testContext.client, testContext.clusterName, testContext.namespace) + if numFailure > 0 { + return fmt.Errorf("%v/%v tests failed", numFailure, len(testsRegistry)) + } return nil } @@ -184,6 +191,18 @@ func (t *testContext) Log(format string, a ...interface{}) { fmt.Fprintf(os.Stdout, fmt.Sprintf("[%s] ", t.clusterName)+format+"\n", a...) } +func (t *testContext) Success(format string, a ...interface{}) { + fmt.Fprintf(os.Stdout, fmt.Sprintf("[%s] ", t.clusterName)+color.GreenString(format, a...)+"\n") +} + +func (t *testContext) Fail(format string, a ...interface{}) { + fmt.Fprintf(os.Stdout, fmt.Sprintf("[%s] ", t.clusterName)+color.RedString(format, a...)+"\n") +} + +func (t *testContext) Warning(format string, a ...interface{}) { + fmt.Fprintf(os.Stdout, fmt.Sprintf("[%s] ", t.clusterName)+color.YellowString(format, a...)+"\n") +} + func (t *testContext) Header(format string, a ...interface{}) { t.Log("-------------------------------------------------------------------------------------------") t.Log(format, a...) From 3087388e11aa425c7c1ff0ed213eb53ca1e9ccd9 Mon Sep 17 00:00:00 2001 From: Kanha gupta Date: Thu, 9 May 2024 04:27:30 +0530 Subject: [PATCH 05/12] Review 3 changes Signed-off-by: Kanha gupta --- .github/workflows/kind.yml | 2 +- pkg/antctl/raw/check/cluster/command.go | 72 +++++++++---------- .../check/cluster/test_checkcniexistence.go | 16 ++--- .../test_checkcontrolplaneavailability.go | 8 +-- .../raw/check/cluster/test_checkk8sversion.go | 4 +- .../check/cluster/test_checkovsloadable.go | 28 +++----- pkg/antctl/raw/check/util.go | 22 +++--- 7 files changed, 70 insertions(+), 82 deletions(-) diff --git a/.github/workflows/kind.yml b/.github/workflows/kind.yml index bd01f15d644..ae97e3ea33b 100644 --- a/.github/workflows/kind.yml +++ b/.github/workflows/kind.yml @@ -777,7 +777,7 @@ jobs: make antctl-linux - name: Run Pre checks run: | - /bin/antctl-linux check cluster + ./bin/antctl-linux check cluster - name: Deploy Antrea run: | kubectl apply -f build/yamls/antrea.yml diff --git a/pkg/antctl/raw/check/cluster/command.go b/pkg/antctl/raw/check/cluster/command.go index a18a89cef09..4a3c2673eb8 100644 --- a/pkg/antctl/raw/check/cluster/command.go +++ b/pkg/antctl/raw/check/cluster/command.go @@ -29,35 +29,23 @@ import ( "k8s.io/utils/ptr" "antrea.io/antrea/pkg/antctl/raw/check" + "antrea.io/antrea/pkg/version" ) func Command() *cobra.Command { - o := newOptions() command := &cobra.Command{ Use: "cluster", Short: "Runs pre installation checks", RunE: func(cmd *cobra.Command, args []string) error { - return Run(o) + return Run() }, } - command.Flags().StringVarP(&o.antreaNamespace, "namespace", "n", o.antreaNamespace, "Configure Namespace in which Antrea is running") return command } -type options struct { - antreaNamespace string -} - -func newOptions() *options { - return &options{ - antreaNamespace: "kube-system", - } -} - const ( - antreaNamespace = "kube-system" testNamespace = "antrea-test" - deploymentName = "check-cluster" + deploymentName = "cluster-checker" podReadyTimeout = 1 * time.Minute ) @@ -72,20 +60,20 @@ func RegisterTest(name string, test Test) { } type testContext struct { - client kubernetes.Interface - config *rest.Config - clusterName string - antreaNamespace string - namespace string + client kubernetes.Interface + config *rest.Config + clusterName string + namespace string + testPod *corev1.Pod } -func Run(o *options) error { +func Run() error { client, config, clusterName, err := check.NewClient() if err != nil { return fmt.Errorf("unable to create Kubernetes client: %s", err) } ctx := context.Background() - testContext := NewTestContext(client, config, clusterName, o) + testContext := NewTestContext(client, config, clusterName) if err := testContext.setup(ctx); err != nil { return err } @@ -116,10 +104,11 @@ func (t *testContext) setup(ctx context.Context) error { } deployment := check.NewDeployment(check.DeploymentParameters{ Name: deploymentName, - Image: "antrea/antrea-agent-ubuntu:latest", + Image: getImageVersion(), Replicas: 1, - Command: []string{"sleep", "infinity"}, - Labels: map[string]string{"app": "check-cluster"}, + Command: []string{"bash", "-c"}, + Args: []string{"trap 'exit 0' SIGTERM; sleep infinity & pid=$!; wait $pid"}, + Labels: map[string]string{"app": "antrea", "component": "cluster-checker"}, HostNetwork: true, VolumeMounts: []corev1.VolumeMount{ {Name: "cni-conf", MountPath: "/etc/cni/net.d"}, @@ -170,20 +159,35 @@ func (t *testContext) setup(ctx context.Context) error { } t.Log("Waiting for Deployment to become ready") - check.WaitForDeploymentsReady(ctx, time.Second, podReadyTimeout, t.client, t.clusterName, t.namespace, deploymentName) + err = check.WaitForDeploymentsReady(ctx, time.Second, podReadyTimeout, t.client, t.clusterName, t.namespace, deploymentName) if err != nil { return fmt.Errorf("error while waiting for Deployment to become ready: %w", err) } + testPods, err := t.client.CoreV1().Pods(t.namespace).List(ctx, metav1.ListOptions{LabelSelector: "component=cluster-checker"}) + if err != nil { + return fmt.Errorf("unable to list test Pod: %s", err) + } + if len(testPods.Items) > 0 { + t.testPod = &testPods.Items[0] + } return nil } -func NewTestContext(client kubernetes.Interface, config *rest.Config, clusterName string, o *options) *testContext { +func getImageVersion() string { + if version.ReleaseStatus == "unreleased" { + return "antrea/antrea-agent-ubuntu:latest" + } else if version.ReleaseStatus == "released" { + return fmt.Sprintf("antrea/antrea-agent-ubuntu:%s", version.GetVersion()) + } + return "" +} + +func NewTestContext(client kubernetes.Interface, config *rest.Config, clusterName string) *testContext { return &testContext{ - client: client, - config: config, - clusterName: clusterName, - antreaNamespace: o.antreaNamespace, - namespace: check.GenerateRandomNamespace(testNamespace), + client: client, + config: config, + clusterName: clusterName, + namespace: check.GenerateRandomNamespace(testNamespace), } } @@ -199,10 +203,6 @@ func (t *testContext) Fail(format string, a ...interface{}) { fmt.Fprintf(os.Stdout, fmt.Sprintf("[%s] ", t.clusterName)+color.RedString(format, a...)+"\n") } -func (t *testContext) Warning(format string, a ...interface{}) { - fmt.Fprintf(os.Stdout, fmt.Sprintf("[%s] ", t.clusterName)+color.YellowString(format, a...)+"\n") -} - func (t *testContext) Header(format string, a ...interface{}) { t.Log("-------------------------------------------------------------------------------------------") t.Log(format, a...) diff --git a/pkg/antctl/raw/check/cluster/test_checkcniexistence.go b/pkg/antctl/raw/check/cluster/test_checkcniexistence.go index 7ab8acd619a..6caf6bcd85f 100644 --- a/pkg/antctl/raw/check/cluster/test_checkcniexistence.go +++ b/pkg/antctl/raw/check/cluster/test_checkcniexistence.go @@ -20,36 +20,30 @@ import ( "sort" "strings" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "antrea.io/antrea/pkg/antctl/raw/check" ) type checkCNIExistence struct{} func init() { - RegisterTest("Check if another CNI is present", &checkCNIExistence{}) + RegisterTest("check-cni-existence", &checkCNIExistence{}) } func (t *checkCNIExistence) Run(ctx context.Context, testContext *testContext) error { - pods, err := testContext.client.CoreV1().Pods(testContext.namespace).List(ctx, metav1.ListOptions{LabelSelector: "name=check-cluster"}) - if err != nil { - return fmt.Errorf("failed to list Pods: %v", err) - } command := []string{"ls", "-1", "/etc/cni/net.d"} - output, _, err := check.ExecInPod(ctx, testContext.client, testContext.config, testContext.namespace, pods.Items[0].Name, "", command) + output, _, err := check.ExecInPod(ctx, testContext.client, testContext.config, testContext.namespace, testContext.testPod.Name, "", command) if err != nil { - testContext.Log("Failed to execute command in Pod: %s, error: %v", pods.Items[0].Name, err) + return fmt.Errorf("Failed to execute command in Pod %s, error: %v", testContext.testPod.Name, err) } outputStr := strings.TrimSpace(output) if outputStr == "" { - testContext.Log("No files present in /etc/cni/net.d in Pod: %s", pods.Items[0].Name) + testContext.Log("No files present in /etc/cni/net.d in Node %s", testContext.testPod.Spec.NodeName) } else { files := strings.Split(outputStr, "\n") sort.Strings(files) if len(files) > 0 { if files[0] < "10-antrea.conflist" { - testContext.Log("Another CNI configuration file with higher priority than Antrea's CNI configuration file found: %s", files[0]) + return fmt.Errorf("Another CNI configuration file with higher priority than Antrea's CNI configuration file found: %s. Ignore this if PolicyOnly mode is enabled.", files[0]) } else if files[0] != "10-antrea.conflist" { testContext.Log("Another CNI configuration file found: %s with Antrea having higher precedence", files[0]) } else { diff --git a/pkg/antctl/raw/check/cluster/test_checkcontrolplaneavailability.go b/pkg/antctl/raw/check/cluster/test_checkcontrolplaneavailability.go index 22e00ba6213..564344fd7f2 100644 --- a/pkg/antctl/raw/check/cluster/test_checkcontrolplaneavailability.go +++ b/pkg/antctl/raw/check/cluster/test_checkcontrolplaneavailability.go @@ -25,7 +25,7 @@ import ( type checkControlPlaneAvailability struct{} func init() { - RegisterTest("Check control plane Nodes availability", &checkControlPlaneAvailability{}) + RegisterTest("check-control-plane-nodes-availability", &checkControlPlaneAvailability{}) } func (t *checkControlPlaneAvailability) Run(ctx context.Context, testContext *testContext) error { @@ -41,11 +41,9 @@ func (t *checkControlPlaneAvailability) Run(ctx context.Context, testContext *te } } if controlPlaneNodes.Len() == 0 { - testContext.Log("No control-plane or master Nodes were found; if installing Antrea in encap mode, some K8s functionalities (API aggregation, apiserver proxy, admission controllers) may be impacted.") + testContext.Log("No control-plane Nodes were found; if installing Antrea in encap mode, some K8s functionalities (API aggregation, apiserver proxy, admission controllers) may be impacted.") } else { - for _, node := range controlPlaneNodes.UnsortedList() { - testContext.Log("Nodes found : %s", node) - } + testContext.Log("control-plane Nodes were found in the cluster.") } return nil } diff --git a/pkg/antctl/raw/check/cluster/test_checkk8sversion.go b/pkg/antctl/raw/check/cluster/test_checkk8sversion.go index 057e2d6c280..482afc576fb 100644 --- a/pkg/antctl/raw/check/cluster/test_checkk8sversion.go +++ b/pkg/antctl/raw/check/cluster/test_checkk8sversion.go @@ -25,7 +25,7 @@ import ( type checkK8sVersion struct{} func init() { - RegisterTest("Check K8s version", &checkK8sVersion{}) + RegisterTest("check-k8s-version", &checkK8sVersion{}) } func (t *checkK8sVersion) Run(ctx context.Context, testContext *testContext) error { @@ -42,7 +42,7 @@ func (t *checkK8sVersion) Run(ctx context.Context, testContext *testContext) err if currentVersion.GTE(minVersion) { testContext.Log("Kubernetes server version is compatible with Antrea. Kubernetes version: %s", serverVersion.GitVersion) } else { - testContext.Log("Kubernetes min version required: 1.19") + return fmt.Errorf("Kubernetes min version required: 1.19") } return nil } diff --git a/pkg/antctl/raw/check/cluster/test_checkovsloadable.go b/pkg/antctl/raw/check/cluster/test_checkovsloadable.go index fcbe1d3449f..2ee302da62a 100644 --- a/pkg/antctl/raw/check/cluster/test_checkovsloadable.go +++ b/pkg/antctl/raw/check/cluster/test_checkovsloadable.go @@ -19,46 +19,40 @@ import ( "fmt" "strings" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "antrea.io/antrea/pkg/antctl/raw/check" ) type checkOVSLoadable struct{} func init() { - RegisterTest("Check if the module openvswitch is loadable", &checkOVSLoadable{}) + RegisterTest("check-if-openvswitch-is-loadable", &checkOVSLoadable{}) } func (c *checkOVSLoadable) Run(ctx context.Context, testContext *testContext) error { - pods, err := testContext.client.CoreV1().Pods(testContext.namespace).List(ctx, metav1.ListOptions{LabelSelector: "name=check-cluster"}) - if err != nil { - return fmt.Errorf("failed to list Pods: %v", err) - } command := []string{ "/bin/sh", "-c", - `path="/lib/modules/$(uname -r)/modules.builtin"; grep -q "openvswitch.ko" "$path"; echo $?`, + "grep -q 'openvswitch.ko' /lib/modules/$(uname -r)/modules.builtin; echo $?", } - stdout, _, err := check.ExecInPod(ctx, testContext.client, testContext.config, testContext.namespace, pods.Items[0].Name, "", command) + stdout, stderr, err := check.ExecInPod(ctx, testContext.client, testContext.config, testContext.namespace, testContext.testPod.Name, "", command) if err != nil { - return fmt.Errorf("error executing command in Pod %s: %v", pods.Items[0].Name, err) + return fmt.Errorf("error executing command in Pod %s: %v", testContext.testPod.Name, err) } if strings.TrimSpace(stdout) == "0" { testContext.Log("The kernel module openvswitch is built-in") } else if strings.TrimSpace(stdout) == "1" { testContext.Log("The kernel module openvswitch is not built-in. Running modprobe command to load the module.") cmd := []string{"modprobe", "openvswitch"} - stdout, stderr, err := check.ExecInPod(ctx, testContext.client, testContext.config, testContext.namespace, pods.Items[0].Name, "", cmd) + _, stderr, err := check.ExecInPod(ctx, testContext.client, testContext.config, testContext.namespace, testContext.testPod.Name, "", cmd) if err != nil { - return fmt.Errorf("error executing modprobe command in Pod %s: %v", pods.Items[0].Name, err) - } - if stderr != "" { - testContext.Log("failed to load the OVS kernel module from the container, try running 'modprobe openvswitch' on your Nodes") - } - if stdout == "" { + return fmt.Errorf("error executing modprobe command in Pod %s: %v", testContext.testPod.Name, err) + } else if stderr != "" { + return fmt.Errorf("failed to load the OVS kernel module from the container %s, try running 'modprobe openvswitch' on your Nodes", stderr) + } else { testContext.Log("openvswitch kernel module loaded successfully") } + } else { + return fmt.Errorf("error encountered while executing modprobe command %s", stderr) } return nil } diff --git a/pkg/antctl/raw/check/util.go b/pkg/antctl/raw/check/util.go index 429702b650c..05cb8ae2cf7 100644 --- a/pkg/antctl/raw/check/util.go +++ b/pkg/antctl/raw/check/util.go @@ -113,10 +113,6 @@ func NewDeployment(p DeploymentParameters) *appsv1.Deployment { p.Replicas = 1 } replicas32 := int32(p.Replicas) - labels := map[string]string{ - "name": p.Name, - "kind": p.Role, - } var ports []corev1.ContainerPort if p.Port > 0 { ports = append(ports, corev1.ContainerPort{ContainerPort: int32(p.Port)}) @@ -125,19 +121,23 @@ func NewDeployment(p DeploymentParameters) *appsv1.Deployment { if p.Port > 0 { env = append(env, corev1.EnvVar{Name: "PORT", Value: fmt.Sprintf("%d", p.Port)}) } + if p.Labels == nil { + p.Labels = make(map[string]string) + p.Labels["name"] = p.Name + } return &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: p.Name, - Labels: labels, + Labels: p.Labels, }, Spec: appsv1.DeploymentSpec{ Replicas: &replicas32, Selector: &metav1.LabelSelector{ - MatchLabels: labels, + MatchLabels: p.Labels, }, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ - Labels: labels, + Labels: p.Labels, }, Spec: corev1.PodSpec{ HostNetwork: p.HostNetwork, @@ -149,6 +149,7 @@ func NewDeployment(p DeploymentParameters) *appsv1.Deployment { Env: env, ImagePullPolicy: corev1.PullIfNotPresent, Command: p.Command, + Args: p.Args, VolumeMounts: p.VolumeMounts, }, }, @@ -168,6 +169,7 @@ type DeploymentParameters struct { Replicas int Port int Command []string + Args []string Affinity *corev1.Affinity Tolerations []corev1.Toleration Labels map[string]string @@ -215,7 +217,7 @@ func GenerateRandomNamespace(baseName string) string { func Teardown(ctx context.Context, client kubernetes.Interface, clusterName string, namespace string) { fmt.Fprintf(os.Stdout, fmt.Sprintf("[%s] ", clusterName)+"Deleting post installation tests setup...\n") client.CoreV1().Namespaces().Delete(ctx, namespace, metav1.DeleteOptions{}) - fmt.Fprintf(os.Stdout, fmt.Sprintf("[%s] ", clusterName)+"Waiting for Namespace %s to disappear \n", namespace) + fmt.Fprintf(os.Stdout, fmt.Sprintf("[%s] ", clusterName)+"Waiting for Namespace %s to be deleted\n", namespace) err := wait.PollUntilContextTimeout(ctx, 2*time.Second, 1*time.Minute, true, func(ctx context.Context) (bool, error) { _, err := client.CoreV1().Namespaces().Get(ctx, namespace, metav1.GetOptions{}) if err != nil { @@ -224,8 +226,8 @@ func Teardown(ctx context.Context, client kubernetes.Interface, clusterName stri return false, nil }) if err != nil { - fmt.Fprintf(os.Stdout, fmt.Sprintf("[%s] ", clusterName)+"Setup deletion failed \n") + fmt.Fprintf(os.Stdout, fmt.Sprintf("[%s] ", clusterName)+"Setup deletion failed\n") } else { - fmt.Fprintf(os.Stdout, fmt.Sprintf("[%s] ", clusterName)+"Setup deletion successful \n") + fmt.Fprintf(os.Stdout, fmt.Sprintf("[%s] ", clusterName)+"Setup deletion successful\n") } } From 0a0801fff9b935f38e33d53257393216440f8afd Mon Sep 17 00:00:00 2001 From: Kanha gupta Date: Fri, 10 May 2024 01:25:48 +0530 Subject: [PATCH 06/12] Review 4 changes Signed-off-by: Kanha gupta --- pkg/antctl/raw/check/cluster/command.go | 18 +++++++------ .../check/cluster/test_checkcniexistence.go | 25 +++++++++---------- pkg/antctl/raw/check/util.go | 4 ++- 3 files changed, 25 insertions(+), 22 deletions(-) diff --git a/pkg/antctl/raw/check/cluster/command.go b/pkg/antctl/raw/check/cluster/command.go index 4a3c2673eb8..72f622b44e1 100644 --- a/pkg/antctl/raw/check/cluster/command.go +++ b/pkg/antctl/raw/check/cluster/command.go @@ -104,7 +104,7 @@ func (t *testContext) setup(ctx context.Context) error { } deployment := check.NewDeployment(check.DeploymentParameters{ Name: deploymentName, - Image: getImageVersion(), + Image: getAntreaAgentImage(), Replicas: 1, Command: []string{"bash", "-c"}, Args: []string{"trap 'exit 0' SIGTERM; sleep infinity & pid=$!; wait $pid"}, @@ -150,6 +150,9 @@ func (t *testContext) setup(ctx context.Context) error { }, }, }, + NodeSelector: map[string]string{ + "kubernetes.io/os": "linux", + }, }) t.Log("Creating Deployment") @@ -167,19 +170,18 @@ func (t *testContext) setup(ctx context.Context) error { if err != nil { return fmt.Errorf("unable to list test Pod: %s", err) } - if len(testPods.Items) > 0 { - t.testPod = &testPods.Items[0] + if len(testPods.Items) == 0 { + return fmt.Errorf("unable to list pods") } + t.testPod = &testPods.Items[0] return nil } -func getImageVersion() string { - if version.ReleaseStatus == "unreleased" { - return "antrea/antrea-agent-ubuntu:latest" - } else if version.ReleaseStatus == "released" { +func getAntreaAgentImage() string { + if version.ReleaseStatus == "released" { return fmt.Sprintf("antrea/antrea-agent-ubuntu:%s", version.GetVersion()) } - return "" + return "antrea/antrea-agent-ubuntu:latest" } func NewTestContext(client kubernetes.Interface, config *rest.Config, clusterName string) *testContext { diff --git a/pkg/antctl/raw/check/cluster/test_checkcniexistence.go b/pkg/antctl/raw/check/cluster/test_checkcniexistence.go index 6caf6bcd85f..5d1a56c7607 100644 --- a/pkg/antctl/raw/check/cluster/test_checkcniexistence.go +++ b/pkg/antctl/raw/check/cluster/test_checkcniexistence.go @@ -35,20 +35,19 @@ func (t *checkCNIExistence) Run(ctx context.Context, testContext *testContext) e if err != nil { return fmt.Errorf("Failed to execute command in Pod %s, error: %v", testContext.testPod.Name, err) } - outputStr := strings.TrimSpace(output) - if outputStr == "" { + files := strings.Fields(output) + if len(files) == 0 { testContext.Log("No files present in /etc/cni/net.d in Node %s", testContext.testPod.Spec.NodeName) - } else { - files := strings.Split(outputStr, "\n") - sort.Strings(files) - if len(files) > 0 { - if files[0] < "10-antrea.conflist" { - return fmt.Errorf("Another CNI configuration file with higher priority than Antrea's CNI configuration file found: %s. Ignore this if PolicyOnly mode is enabled.", files[0]) - } else if files[0] != "10-antrea.conflist" { - testContext.Log("Another CNI configuration file found: %s with Antrea having higher precedence", files[0]) - } else { - testContext.Log("Antrea's CNI configuration file already present: %s", files[0]) - } + return nil + } + sort.Strings(files) + if len(files) > 0 { + if files[0] < "10-antrea.conflist" { + return fmt.Errorf("Another CNI configuration file with higher priority than Antrea's CNI configuration file found: %s; this may be expected if networkPolicyOnly mode is enabled", files[0]) + } else if files[0] != "10-antrea.conflist" { + testContext.Log("Another CNI configuration file found: %s with Antrea having higher precedence", files[0]) + } else { + testContext.Log("Antrea's CNI configuration file already present: %s", files[0]) } } return nil diff --git a/pkg/antctl/raw/check/util.go b/pkg/antctl/raw/check/util.go index 05cb8ae2cf7..1652ea9835a 100644 --- a/pkg/antctl/raw/check/util.go +++ b/pkg/antctl/raw/check/util.go @@ -140,7 +140,8 @@ func NewDeployment(p DeploymentParameters) *appsv1.Deployment { Labels: p.Labels, }, Spec: corev1.PodSpec{ - HostNetwork: p.HostNetwork, + HostNetwork: p.HostNetwork, + NodeSelector: p.NodeSelector, Containers: []corev1.Container{ { Name: p.Name, @@ -176,6 +177,7 @@ type DeploymentParameters struct { VolumeMounts []corev1.VolumeMount Volumes []corev1.Volume HostNetwork bool + NodeSelector map[string]string } func WaitForDeploymentsReady(ctx context.Context, From 66f0fe0221e3ce65294d6a8c3073851fe9e9b21a Mon Sep 17 00:00:00 2001 From: Kanha gupta Date: Sat, 11 May 2024 03:06:27 +0530 Subject: [PATCH 07/12] Review 5 changes Signed-off-by: Kanha gupta --- pkg/antctl/raw/check/cluster/command.go | 38 ++++++++++++++----- .../check/cluster/test_checkcniexistence.go | 16 ++++---- .../test_checkcontrolplaneavailability.go | 2 +- .../check/cluster/test_checkovsloadable.go | 4 +- 4 files changed, 38 insertions(+), 22 deletions(-) diff --git a/pkg/antctl/raw/check/cluster/command.go b/pkg/antctl/raw/check/cluster/command.go index 72f622b44e1..cfe20788ac6 100644 --- a/pkg/antctl/raw/check/cluster/command.go +++ b/pkg/antctl/raw/check/cluster/command.go @@ -16,6 +16,7 @@ package cluster import ( "context" + "errors" "fmt" "os" "time" @@ -44,11 +45,23 @@ func Command() *cobra.Command { } const ( - testNamespace = "antrea-test" - deploymentName = "cluster-checker" - podReadyTimeout = 1 * time.Minute + testNamespacePrefix = "antrea-test" + deploymentName = "cluster-checker" + podReadyTimeout = 1 * time.Minute ) +type uncertainError struct { + reason string +} + +func (e uncertainError) Error() string { + return fmt.Sprintf("test results are uncertain: %s", e.reason) +} + +func newUncertainError(reason string, a ...interface{}) uncertainError { + return uncertainError{reason: fmt.Sprintf(reason, a...)} +} + type Test interface { Run(ctx context.Context, testContext *testContext) error } @@ -77,10 +90,14 @@ func Run() error { if err := testContext.setup(ctx); err != nil { return err } - var numSuccess, numFailure int + var numSuccess, numFailure, numSkipped int for name, test := range testsRegistry { testContext.Header("Running test: %s", name) if err := test.Run(ctx, testContext); err != nil { + if errors.As(err, new(uncertainError)) { + testContext.Warning("Test %s was skipped: %v", name, err) + numSkipped++ + } testContext.Fail("Test %s failed: %v", name, err) numFailure++ } else { @@ -88,7 +105,7 @@ func Run() error { numSuccess++ } } - testContext.Log("Test finished: %v tests succeeded, %v tests failed ", numSuccess, numFailure) + testContext.Log("Test finished: %v tests succeeded, %v tests failed, %v tests were skipped", numSuccess, numFailure, numSkipped) check.Teardown(ctx, testContext.client, testContext.clusterName, testContext.namespace) if numFailure > 0 { return fmt.Errorf("%v/%v tests failed", numFailure, len(testsRegistry)) @@ -168,10 +185,7 @@ func (t *testContext) setup(ctx context.Context) error { } testPods, err := t.client.CoreV1().Pods(t.namespace).List(ctx, metav1.ListOptions{LabelSelector: "component=cluster-checker"}) if err != nil { - return fmt.Errorf("unable to list test Pod: %s", err) - } - if len(testPods.Items) == 0 { - return fmt.Errorf("unable to list pods") + return fmt.Errorf("no pod found for test Deployment") } t.testPod = &testPods.Items[0] return nil @@ -189,7 +203,7 @@ func NewTestContext(client kubernetes.Interface, config *rest.Config, clusterNam client: client, config: config, clusterName: clusterName, - namespace: check.GenerateRandomNamespace(testNamespace), + namespace: check.GenerateRandomNamespace(testNamespacePrefix), } } @@ -205,6 +219,10 @@ func (t *testContext) Fail(format string, a ...interface{}) { fmt.Fprintf(os.Stdout, fmt.Sprintf("[%s] ", t.clusterName)+color.RedString(format, a...)+"\n") } +func (t *testContext) Warning(format string, a ...interface{}) { + fmt.Fprintf(os.Stdout, fmt.Sprintf("[%s] ", t.clusterName)+color.YellowString(format, a...)+"\n") +} + func (t *testContext) Header(format string, a ...interface{}) { t.Log("-------------------------------------------------------------------------------------------") t.Log(format, a...) diff --git a/pkg/antctl/raw/check/cluster/test_checkcniexistence.go b/pkg/antctl/raw/check/cluster/test_checkcniexistence.go index 5d1a56c7607..9de591887c2 100644 --- a/pkg/antctl/raw/check/cluster/test_checkcniexistence.go +++ b/pkg/antctl/raw/check/cluster/test_checkcniexistence.go @@ -33,7 +33,7 @@ func (t *checkCNIExistence) Run(ctx context.Context, testContext *testContext) e command := []string{"ls", "-1", "/etc/cni/net.d"} output, _, err := check.ExecInPod(ctx, testContext.client, testContext.config, testContext.namespace, testContext.testPod.Name, "", command) if err != nil { - return fmt.Errorf("Failed to execute command in Pod %s, error: %v", testContext.testPod.Name, err) + return fmt.Errorf("failed to execute command in Pod %s, error: %v", testContext.testPod.Name, err) } files := strings.Fields(output) if len(files) == 0 { @@ -41,14 +41,12 @@ func (t *checkCNIExistence) Run(ctx context.Context, testContext *testContext) e return nil } sort.Strings(files) - if len(files) > 0 { - if files[0] < "10-antrea.conflist" { - return fmt.Errorf("Another CNI configuration file with higher priority than Antrea's CNI configuration file found: %s; this may be expected if networkPolicyOnly mode is enabled", files[0]) - } else if files[0] != "10-antrea.conflist" { - testContext.Log("Another CNI configuration file found: %s with Antrea having higher precedence", files[0]) - } else { - testContext.Log("Antrea's CNI configuration file already present: %s", files[0]) - } + if files[0] < "10-antrea.conflist" { + return newUncertainError("another CNI configuration file with higher priority than Antrea's CNI configuration file found: %s; this may be expected if networkPolicyOnly mode is enabled", files[0]) + } else if files[0] != "10-antrea.conflist" { + testContext.Log("Another CNI configuration file found: %s with Antrea having higher precedence", files[0]) + } else { + testContext.Log("Antrea's CNI configuration file already present: %s", files[0]) } return nil } diff --git a/pkg/antctl/raw/check/cluster/test_checkcontrolplaneavailability.go b/pkg/antctl/raw/check/cluster/test_checkcontrolplaneavailability.go index 564344fd7f2..a4a7b1f8295 100644 --- a/pkg/antctl/raw/check/cluster/test_checkcontrolplaneavailability.go +++ b/pkg/antctl/raw/check/cluster/test_checkcontrolplaneavailability.go @@ -41,7 +41,7 @@ func (t *checkControlPlaneAvailability) Run(ctx context.Context, testContext *te } } if controlPlaneNodes.Len() == 0 { - testContext.Log("No control-plane Nodes were found; if installing Antrea in encap mode, some K8s functionalities (API aggregation, apiserver proxy, admission controllers) may be impacted.") + return newUncertainError("No control-plane Nodes were found; if installing Antrea in encap mode, some K8s functionalities (API aggregation, apiserver proxy, admission controllers) may be impacted.") } else { testContext.Log("control-plane Nodes were found in the cluster.") } diff --git a/pkg/antctl/raw/check/cluster/test_checkovsloadable.go b/pkg/antctl/raw/check/cluster/test_checkovsloadable.go index 2ee302da62a..020016622e6 100644 --- a/pkg/antctl/raw/check/cluster/test_checkovsloadable.go +++ b/pkg/antctl/raw/check/cluster/test_checkovsloadable.go @@ -47,12 +47,12 @@ func (c *checkOVSLoadable) Run(ctx context.Context, testContext *testContext) er if err != nil { return fmt.Errorf("error executing modprobe command in Pod %s: %v", testContext.testPod.Name, err) } else if stderr != "" { - return fmt.Errorf("failed to load the OVS kernel module from the container %s, try running 'modprobe openvswitch' on your Nodes", stderr) + return fmt.Errorf("failed to load the OVS kernel module: %s, try running 'modprobe openvswitch' on your Nodes", stderr) } else { testContext.Log("openvswitch kernel module loaded successfully") } } else { - return fmt.Errorf("error encountered while executing modprobe command %s", stderr) + return fmt.Errorf("error encountered while check if cni is existent - stderr: %s", stderr) } return nil } From 9e0141d29970bf594ab65fd02b6e87f41ae0a578 Mon Sep 17 00:00:00 2001 From: Kanha gupta Date: Sat, 11 May 2024 11:02:16 +0530 Subject: [PATCH 08/12] Review 6 changes Signed-off-by: Kanha gupta --- pkg/antctl/raw/check/cluster/command.go | 10 +++++----- .../cluster/test_checkcontrolplaneavailability.go | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/antctl/raw/check/cluster/command.go b/pkg/antctl/raw/check/cluster/command.go index cfe20788ac6..e571ea96b01 100644 --- a/pkg/antctl/raw/check/cluster/command.go +++ b/pkg/antctl/raw/check/cluster/command.go @@ -55,7 +55,7 @@ type uncertainError struct { } func (e uncertainError) Error() string { - return fmt.Sprintf("test results are uncertain: %s", e.reason) + return e.reason } func newUncertainError(reason string, a ...interface{}) uncertainError { @@ -90,13 +90,13 @@ func Run() error { if err := testContext.setup(ctx); err != nil { return err } - var numSuccess, numFailure, numSkipped int + var numSuccess, numFailure, numUncertain int for name, test := range testsRegistry { testContext.Header("Running test: %s", name) if err := test.Run(ctx, testContext); err != nil { if errors.As(err, new(uncertainError)) { - testContext.Warning("Test %s was skipped: %v", name, err) - numSkipped++ + testContext.Warning("Test %s was uncertain: %v", name, err) + numUncertain++ } testContext.Fail("Test %s failed: %v", name, err) numFailure++ @@ -105,7 +105,7 @@ func Run() error { numSuccess++ } } - testContext.Log("Test finished: %v tests succeeded, %v tests failed, %v tests were skipped", numSuccess, numFailure, numSkipped) + testContext.Log("Test finished: %v tests succeeded, %v tests failed, %v tests were uncertain", numSuccess, numFailure, numUncertain) check.Teardown(ctx, testContext.client, testContext.clusterName, testContext.namespace) if numFailure > 0 { return fmt.Errorf("%v/%v tests failed", numFailure, len(testsRegistry)) diff --git a/pkg/antctl/raw/check/cluster/test_checkcontrolplaneavailability.go b/pkg/antctl/raw/check/cluster/test_checkcontrolplaneavailability.go index a4a7b1f8295..89cfd8e9182 100644 --- a/pkg/antctl/raw/check/cluster/test_checkcontrolplaneavailability.go +++ b/pkg/antctl/raw/check/cluster/test_checkcontrolplaneavailability.go @@ -41,7 +41,7 @@ func (t *checkControlPlaneAvailability) Run(ctx context.Context, testContext *te } } if controlPlaneNodes.Len() == 0 { - return newUncertainError("No control-plane Nodes were found; if installing Antrea in encap mode, some K8s functionalities (API aggregation, apiserver proxy, admission controllers) may be impacted.") + return newUncertainError("no control-plane Nodes were found; if installing Antrea in encap mode, some K8s functionalities (API aggregation, apiserver proxy, admission controllers) may be impacted") } else { testContext.Log("control-plane Nodes were found in the cluster.") } From 7651313b5c0f874bfc0bb7e0bb53c4f4a3b668ad Mon Sep 17 00:00:00 2001 From: Kanha gupta Date: Sat, 11 May 2024 11:09:30 +0530 Subject: [PATCH 09/12] merge main Signed-off-by: Kanha gupta --- pkg/antctl/raw/check/cluster/command.go | 5 +++-- pkg/antctl/raw/check/installation/command.go | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/pkg/antctl/raw/check/cluster/command.go b/pkg/antctl/raw/check/cluster/command.go index e571ea96b01..e6c80aaf82c 100644 --- a/pkg/antctl/raw/check/cluster/command.go +++ b/pkg/antctl/raw/check/cluster/command.go @@ -97,9 +97,10 @@ func Run() error { if errors.As(err, new(uncertainError)) { testContext.Warning("Test %s was uncertain: %v", name, err) numUncertain++ + } else { + testContext.Fail("Test %s failed: %v", name, err) + numFailure++ } - testContext.Fail("Test %s failed: %v", name, err) - numFailure++ } else { testContext.Success("Test %s passed", name) numSuccess++ diff --git a/pkg/antctl/raw/check/installation/command.go b/pkg/antctl/raw/check/installation/command.go index c1612693833..82b550ac1cc 100644 --- a/pkg/antctl/raw/check/installation/command.go +++ b/pkg/antctl/raw/check/installation/command.go @@ -235,7 +235,7 @@ func (t *testContext) setup(ctx context.Context) error { if err != nil { return fmt.Errorf("unable to create Deployment %s: %s", clientDeploymentName, err) } - echoOtherNodeDeployment := newDeployment(deploymentParameters{ + echoOtherNodeDeployment := check.NewDeployment(check.DeploymentParameters{ Name: echoOtherNodeDeploymentName, Role: kindEchoName, Port: 80, From 548a115a434eabfe53e793425fc49aa9fe5c8aac Mon Sep 17 00:00:00 2001 From: Kanha gupta Date: Mon, 13 May 2024 18:08:41 +0530 Subject: [PATCH 10/12] few minor improvements Signed-off-by: Kanha gupta --- pkg/antctl/raw/check/cluster/test_checkcniexistence.go | 2 +- pkg/antctl/raw/check/cluster/test_checkk8sversion.go | 4 ++-- pkg/antctl/raw/check/cluster/test_checkovsloadable.go | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/antctl/raw/check/cluster/test_checkcniexistence.go b/pkg/antctl/raw/check/cluster/test_checkcniexistence.go index 9de591887c2..11a38024616 100644 --- a/pkg/antctl/raw/check/cluster/test_checkcniexistence.go +++ b/pkg/antctl/raw/check/cluster/test_checkcniexistence.go @@ -33,7 +33,7 @@ func (t *checkCNIExistence) Run(ctx context.Context, testContext *testContext) e command := []string{"ls", "-1", "/etc/cni/net.d"} output, _, err := check.ExecInPod(ctx, testContext.client, testContext.config, testContext.namespace, testContext.testPod.Name, "", command) if err != nil { - return fmt.Errorf("failed to execute command in Pod %s, error: %v", testContext.testPod.Name, err) + return fmt.Errorf("failed to execute command in Pod %s, error: %w", testContext.testPod.Name, err) } files := strings.Fields(output) if len(files) == 0 { diff --git a/pkg/antctl/raw/check/cluster/test_checkk8sversion.go b/pkg/antctl/raw/check/cluster/test_checkk8sversion.go index 482afc576fb..277de84b501 100644 --- a/pkg/antctl/raw/check/cluster/test_checkk8sversion.go +++ b/pkg/antctl/raw/check/cluster/test_checkk8sversion.go @@ -32,11 +32,11 @@ func (t *checkK8sVersion) Run(ctx context.Context, testContext *testContext) err discoveryClient := testContext.client.Discovery() serverVersion, err := discoveryClient.ServerVersion() if err != nil { - return fmt.Errorf("error getting server version: %v", err) + return fmt.Errorf("error getting server version: %w", err) } currentVersion, err := semver.Parse(strings.TrimPrefix(serverVersion.GitVersion, "v")) if err != nil { - return fmt.Errorf("error parsing server version: %v", err) + return fmt.Errorf("error parsing server version: %w", err) } minVersion, _ := semver.Parse("1.19") if currentVersion.GTE(minVersion) { diff --git a/pkg/antctl/raw/check/cluster/test_checkovsloadable.go b/pkg/antctl/raw/check/cluster/test_checkovsloadable.go index 020016622e6..a27d501eee3 100644 --- a/pkg/antctl/raw/check/cluster/test_checkovsloadable.go +++ b/pkg/antctl/raw/check/cluster/test_checkovsloadable.go @@ -36,7 +36,7 @@ func (c *checkOVSLoadable) Run(ctx context.Context, testContext *testContext) er } stdout, stderr, err := check.ExecInPod(ctx, testContext.client, testContext.config, testContext.namespace, testContext.testPod.Name, "", command) if err != nil { - return fmt.Errorf("error executing command in Pod %s: %v", testContext.testPod.Name, err) + return fmt.Errorf("error executing command in Pod %s: %w", testContext.testPod.Name, err) } if strings.TrimSpace(stdout) == "0" { testContext.Log("The kernel module openvswitch is built-in") @@ -45,14 +45,14 @@ func (c *checkOVSLoadable) Run(ctx context.Context, testContext *testContext) er cmd := []string{"modprobe", "openvswitch"} _, stderr, err := check.ExecInPod(ctx, testContext.client, testContext.config, testContext.namespace, testContext.testPod.Name, "", cmd) if err != nil { - return fmt.Errorf("error executing modprobe command in Pod %s: %v", testContext.testPod.Name, err) + return fmt.Errorf("error executing modprobe command in Pod %s: %w", testContext.testPod.Name, err) } else if stderr != "" { return fmt.Errorf("failed to load the OVS kernel module: %s, try running 'modprobe openvswitch' on your Nodes", stderr) } else { testContext.Log("openvswitch kernel module loaded successfully") } } else { - return fmt.Errorf("error encountered while check if cni is existent - stderr: %s", stderr) + return fmt.Errorf("error encountered while checking if openvswitch module is built-in - stderr: %s", stderr) } return nil } From 0f1d7e17f5dbb6213d38e367811d6becd0ae60ad Mon Sep 17 00:00:00 2001 From: Kanha gupta Date: Wed, 15 May 2024 00:45:24 +0530 Subject: [PATCH 11/12] Added container security capability Signed-off-by: Kanha gupta --- pkg/antctl/raw/check/cluster/command.go | 5 ++++ pkg/antctl/raw/check/util.go | 32 +++++++++++++------------ 2 files changed, 22 insertions(+), 15 deletions(-) diff --git a/pkg/antctl/raw/check/cluster/command.go b/pkg/antctl/raw/check/cluster/command.go index e6c80aaf82c..07c3ae313ce 100644 --- a/pkg/antctl/raw/check/cluster/command.go +++ b/pkg/antctl/raw/check/cluster/command.go @@ -171,6 +171,11 @@ func (t *testContext) setup(ctx context.Context) error { NodeSelector: map[string]string{ "kubernetes.io/os": "linux", }, + SecurityContext: &corev1.SecurityContext{ + Capabilities: &corev1.Capabilities{ + Add: []corev1.Capability{"SYS_MODULE"}, + }, + }, }) t.Log("Creating Deployment") diff --git a/pkg/antctl/raw/check/util.go b/pkg/antctl/raw/check/util.go index 1652ea9835a..15e00a48fc4 100644 --- a/pkg/antctl/raw/check/util.go +++ b/pkg/antctl/raw/check/util.go @@ -152,6 +152,7 @@ func NewDeployment(p DeploymentParameters) *appsv1.Deployment { Command: p.Command, Args: p.Args, VolumeMounts: p.VolumeMounts, + SecurityContext: p.SecurityContext, }, }, Tolerations: p.Tolerations, @@ -164,20 +165,21 @@ func NewDeployment(p DeploymentParameters) *appsv1.Deployment { } type DeploymentParameters struct { - Name string - Role string - Image string - Replicas int - Port int - Command []string - Args []string - Affinity *corev1.Affinity - Tolerations []corev1.Toleration - Labels map[string]string - VolumeMounts []corev1.VolumeMount - Volumes []corev1.Volume - HostNetwork bool - NodeSelector map[string]string + Name string + Role string + Image string + Replicas int + Port int + Command []string + Args []string + Affinity *corev1.Affinity + Tolerations []corev1.Toleration + Labels map[string]string + VolumeMounts []corev1.VolumeMount + Volumes []corev1.Volume + HostNetwork bool + NodeSelector map[string]string + SecurityContext *corev1.SecurityContext } func WaitForDeploymentsReady(ctx context.Context, @@ -217,7 +219,7 @@ func GenerateRandomNamespace(baseName string) string { } func Teardown(ctx context.Context, client kubernetes.Interface, clusterName string, namespace string) { - fmt.Fprintf(os.Stdout, fmt.Sprintf("[%s] ", clusterName)+"Deleting post installation tests setup...\n") + fmt.Fprintf(os.Stdout, fmt.Sprintf("[%s] ", clusterName)+"Deleting installation tests setup...\n") client.CoreV1().Namespaces().Delete(ctx, namespace, metav1.DeleteOptions{}) fmt.Fprintf(os.Stdout, fmt.Sprintf("[%s] ", clusterName)+"Waiting for Namespace %s to be deleted\n", namespace) err := wait.PollUntilContextTimeout(ctx, 2*time.Second, 1*time.Minute, true, func(ctx context.Context) (bool, error) { From 01605a3b06336558a512ff29ed38c0396c89336b Mon Sep 17 00:00:00 2001 From: Kanha gupta Date: Wed, 15 May 2024 02:11:31 +0530 Subject: [PATCH 12/12] Error handling for labels Signed-off-by: Kanha gupta --- pkg/antctl/raw/check/util.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/antctl/raw/check/util.go b/pkg/antctl/raw/check/util.go index 15e00a48fc4..ba589a0ab26 100644 --- a/pkg/antctl/raw/check/util.go +++ b/pkg/antctl/raw/check/util.go @@ -123,8 +123,9 @@ func NewDeployment(p DeploymentParameters) *appsv1.Deployment { } if p.Labels == nil { p.Labels = make(map[string]string) - p.Labels["name"] = p.Name } + p.Labels["name"] = p.Name + p.Labels["kind"] = p.Role return &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: p.Name,