diff --git a/cmd/alerts/silence/add_silence.go b/cmd/alerts/silence/add_silence.go index 3bdf9a98..1de538b5 100644 --- a/cmd/alerts/silence/add_silence.go +++ b/cmd/alerts/silence/add_silence.go @@ -36,11 +36,9 @@ func NewCmdAddSilence() *cobra.Command { } cmd.Flags().StringSliceVar(&addSilenceCmd.alertID, "alertname", []string{}, "alertname (comma-separated)") - cmd.Flags().StringVarP(&addSilenceCmd.comment, "comment", "c", "", "add comment about silence") - cmd.Flags().StringVarP(&addSilenceCmd.duration, "duration", "d", "15d", "add duration for silence") //default duration set to 15 days - cmd.Flags().BoolVarP(&addSilenceCmd.all, "all", "a", false, "add silences for all alert") - cmd.Flags().StringVar(&addSilenceCmd.reason, "reason", "", "The reason for this command, which requires elevation, to be run (usualy an OHSS or PD ticket)") - _ = cmd.MarkFlagRequired("reason") + cmd.Flags().StringVarP(&addSilenceCmd.comment, "comment", "c", "Adding silence using the osdctl alert command", "add comment about silence") + cmd.Flags().StringVarP(&addSilenceCmd.duration, "duration", "d", "15d", "adding duration for silence") //default duration set to 15 days + cmd.Flags().BoolVarP(&addSilenceCmd.all, "all", "a", false, "adding silences for all alert") return cmd } @@ -87,7 +85,8 @@ func AddAllSilence(clusterID, duration, comment, username, clustername string, k output, err := ExecInPod(kubeconfig, clientset, addCmd) if err != nil { - fmt.Println(err) + log.Fatal("Exiting the program") + return } formattedOutput := strings.Replace(output, "\n", " ", -1) @@ -109,7 +108,8 @@ func AddAlertNameSilence(alertID []string, duration, comment, username string, k output, err := ExecInPod(kubeconfig, clientset, addCmd) if err != nil { - fmt.Println(err) + log.Fatal("Exiting the program") + return } formattedOutput := strings.Replace(output, "\n", " ", -1) diff --git a/cmd/alerts/silence/common.go b/cmd/alerts/silence/common.go index 85bbceda..010d66ff 100644 --- a/cmd/alerts/silence/common.go +++ b/cmd/alerts/silence/common.go @@ -16,14 +16,37 @@ const ( AccountNamespace = "openshift-monitoring" ContainerName = "alertmanager" LocalHostUrl = "http://localhost:9093" - PodName = "alertmanager-main-0" + PrimaryPod = "alertmanager-main-0" + SecondaryPod = "alertmanager-main-1" ) +// ExecInPod is designed to execute a command inside a Kubernetes pod and capture its output. func ExecInPod(kubeconfig *rest.Config, clientset *kubernetes.Clientset, cmd []string) (string, error) { + var cmdOutput string + var err error - req := clientset.CoreV1().RESTClient().Post().Resource("pods").Name(PodName). - Namespace(AccountNamespace).SubResource("exec") + cmdOutput, err = ExecWithPod(kubeconfig, clientset, PrimaryPod, cmd) + if err == nil { + return cmdOutput, nil + } + + fmt.Printf("Execution with alertmanager-main-0 failed: %v\n", err) + + fmt.Println("Attempting with alertmanger-main-1") + cmdOutput, err = ExecWithPod(kubeconfig, clientset, SecondaryPod, cmd) + if err == nil { + return cmdOutput, nil + } + + fmt.Printf("Execution with alertmanager-main-1 failed: %v\n", err) + fmt.Println("Execution Failed with alertmanager-main-0 and alertmanager-main-1. Please put silence manually") + return "", err +} + +func ExecWithPod(kubeconfig *rest.Config, clientset *kubernetes.Clientset, podName string, cmd []string) (string, error) { + req := clientset.CoreV1().RESTClient().Post().Resource("pods").Name(podName). + Namespace(AccountNamespace).SubResource("exec") option := &corev1.PodExecOptions{ Container: ContainerName, Command: cmd, @@ -37,23 +60,20 @@ func ExecInPod(kubeconfig *rest.Config, clientset *kubernetes.Clientset, cmd []s exec, err := remotecommand.NewSPDYExecutor(kubeconfig, "POST", req.URL()) if err != nil { - return "", fmt.Errorf("failed to create SPDY executor: %w", err) + return "", fmt.Errorf("failed to create executor: %w", err) } capture := &cluster.LogCapture{} errorCapture := &cluster.LogCapture{} - err = exec.StreamWithContext(context.TODO(), remotecommand.StreamOptions{ Stdin: nil, Stdout: capture, Stderr: errorCapture, Tty: false, }) - if err != nil { return "", fmt.Errorf("failed to stream with context: %w", err) } - cmdOutput := capture.GetStdOut() - return cmdOutput, nil + return capture.GetStdOut(), nil }