Merge pull request #545 from itsmitul9/OSD-22074

[OSD-22074] Bug Fix for osdctl alerts silence add fails to run pod

cmd/alerts/silence/add_silence.go

@@ -36,11 +36,9 @@ func NewCmdAddSilence() *cobra.Command {
 	}
 
 	cmd.Flags().StringSliceVar(&addSilenceCmd.alertID, "alertname", []string{}, "alertname (comma-separated)")
-	cmd.Flags().StringVarP(&addSilenceCmd.comment, "comment", "c", "", "add comment about silence")
-	cmd.Flags().StringVarP(&addSilenceCmd.duration, "duration", "d", "15d", "add duration for silence") //default duration set to 15 days
-	cmd.Flags().BoolVarP(&addSilenceCmd.all, "all", "a", false, "add silences for all alert")
-	cmd.Flags().StringVar(&addSilenceCmd.reason, "reason", "", "The reason for this command, which requires elevation, to be run (usualy an OHSS or PD ticket)")
-	_ = cmd.MarkFlagRequired("reason")
+	cmd.Flags().StringVarP(&addSilenceCmd.comment, "comment", "c", "Adding silence using the osdctl alert command", "add comment about silence")
+	cmd.Flags().StringVarP(&addSilenceCmd.duration, "duration", "d", "15d", "adding duration for silence") //default duration set to 15 days
+	cmd.Flags().BoolVarP(&addSilenceCmd.all, "all", "a", false, "adding silences for all alert")
 
 	return cmd
 }
@@ -87,7 +85,8 @@ func AddAllSilence(clusterID, duration, comment, username, clustername string, k
 
 	output, err := ExecInPod(kubeconfig, clientset, addCmd)
 	if err != nil {
-		fmt.Println(err)
+		log.Fatal("Exiting the program")
+		return
 	}
 
 	formattedOutput := strings.Replace(output, "\n", " ", -1)
@@ -109,7 +108,8 @@ func AddAlertNameSilence(alertID []string, duration, comment, username string, k
 
 		output, err := ExecInPod(kubeconfig, clientset, addCmd)
 		if err != nil {
-			fmt.Println(err)
+			log.Fatal("Exiting the program")
+			return
 		}
 
 		formattedOutput := strings.Replace(output, "\n", " ", -1)

cmd/alerts/silence/common.go

@@ -16,14 +16,37 @@ const (
 	AccountNamespace = "openshift-monitoring"
 	ContainerName    = "alertmanager"
 	LocalHostUrl     = "http://localhost:9093"
-	PodName          = "alertmanager-main-0"
+	PrimaryPod       = "alertmanager-main-0"
+	SecondaryPod     = "alertmanager-main-1"
 )
 
+// ExecInPod is designed to execute a command inside a Kubernetes pod and capture its output.
 func ExecInPod(kubeconfig *rest.Config, clientset *kubernetes.Clientset, cmd []string) (string, error) {
+	var cmdOutput string
+	var err error
 
-	req := clientset.CoreV1().RESTClient().Post().Resource("pods").Name(PodName).
-		Namespace(AccountNamespace).SubResource("exec")
+	cmdOutput, err = ExecWithPod(kubeconfig, clientset, PrimaryPod, cmd)
+	if err == nil {
+		return cmdOutput, nil
+	}
+
+	fmt.Printf("Execution with alertmanager-main-0 failed: %v\n", err)
+
+	fmt.Println("Attempting with alertmanger-main-1")
+	cmdOutput, err = ExecWithPod(kubeconfig, clientset, SecondaryPod, cmd)
+	if err == nil {
+		return cmdOutput, nil
+	}
+
+	fmt.Printf("Execution with alertmanager-main-1 failed: %v\n", err)
 
+	fmt.Println("Execution Failed with alertmanager-main-0 and alertmanager-main-1. Please put silence manually")
+	return "", err
+}
+
+func ExecWithPod(kubeconfig *rest.Config, clientset *kubernetes.Clientset, podName string, cmd []string) (string, error) {
+	req := clientset.CoreV1().RESTClient().Post().Resource("pods").Name(podName).
+		Namespace(AccountNamespace).SubResource("exec")
 	option := &corev1.PodExecOptions{
 		Container: ContainerName,
 		Command:   cmd,
@@ -37,23 +60,20 @@ func ExecInPod(kubeconfig *rest.Config, clientset *kubernetes.Clientset, cmd []s
 
 	exec, err := remotecommand.NewSPDYExecutor(kubeconfig, "POST", req.URL())
 	if err != nil {
-		return "", fmt.Errorf("failed to create SPDY executor: %w", err)
+		return "", fmt.Errorf("failed to create executor: %w", err)
 	}
 
 	capture := &cluster.LogCapture{}
 	errorCapture := &cluster.LogCapture{}
-
 	err = exec.StreamWithContext(context.TODO(), remotecommand.StreamOptions{
 		Stdin:  nil,
 		Stdout: capture,
 		Stderr: errorCapture,
 		Tty:    false,
 	})
-
 	if err != nil {
 		return "", fmt.Errorf("failed to stream with context: %w", err)
 	}
 
-	cmdOutput := capture.GetStdOut()
-	return cmdOutput, nil
+	return capture.GetStdOut(), nil
 }