-
Notifications
You must be signed in to change notification settings - Fork 1.4k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
This PR added a subcommand `velero debug`, which leverages `crashd` to collect logs and specs of velero server components and bundle them in a tarball. Signed-off-by: Daniel Jiang <jiangd@vmware.com>
- Loading branch information
1 parent
ed5809b
commit 4c4e1af
Showing
7 changed files
with
182 additions
and
9 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| Implement velero debug |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,25 @@ | ||
| def capture_backup_logs(namespace): | ||
| if args.backup: | ||
| kube_capture(what="objects", namespaces=[namespace], groups=['velero.io'], kinds=['backup'], names=[args.backup]) | ||
| backupLogsCmd = "velero --namespace={} backup logs {}".format(namespace, args.backup) | ||
| capture_local(cmd=backupLogsCmd, file_name="backup_{}.log".format(args.backup)) | ||
| def capture_restore_logs(namespace): | ||
| if args.restore: | ||
| kube_capture(what="objects", namespaces=[namespace], groups=['velero.io'], kinds=['restore'], names=[args.restore]) | ||
| restoreLogsCmd = "velero --namespace={} restore logs {}".format(namespace, args.restore) | ||
| capture_local(cmd=restoreLogsCmd, file_name="restore_{}.log".format(args.restore)) | ||
|
|
||
| ns = args.namespace if args.namespace else "velero" | ||
| basedir = args.basedir if args.basedir else os.home | ||
| output = args.output if args.output else "bundle.tar.gz" | ||
| # Working dir for writing during script execution | ||
| crshd = crashd_config(workdir="{0}/velero-bundle".format(basedir)) | ||
| set_defaults(kube_config(path=args.kubeconfig)) | ||
| capture_local(cmd="velero version -n {}".format(ns), file_name="version.txt") | ||
| capture_backup_logs(ns) | ||
| capture_restore_logs(ns) | ||
| kube_capture(what="logs", namespaces=[ns]) | ||
| kube_capture(what="objects", namespaces=[ns], groups=['velero.io'], kinds=['backupstoragelocation', 'podvolumebackup', 'podvolumerestore']) | ||
| archive(output_file=output, source_paths=[crshd.workdir]) | ||
|
|
||
|
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,140 @@ | ||
| /* | ||
| Copyright 2017 the Velero contributors. | ||
| Licensed under the Apache License, Version 2.0 (the "License"); | ||
| you may not use this file except in compliance with the License. | ||
| You may obtain a copy of the License at | ||
| http://www.apache.org/licenses/LICENSE-2.0 | ||
| Unless required by applicable law or agreed to in writing, software | ||
| distributed under the License is distributed on an "AS IS" BASIS, | ||
| WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| See the License for the specific language governing permissions and | ||
| limitations under the License. | ||
| */ | ||
|
|
||
| package debug | ||
|
|
||
| import ( | ||
| _ "embed" | ||
| "fmt" | ||
| "io/ioutil" | ||
| "os" | ||
| "path/filepath" | ||
|
|
||
| "github.com/spf13/cobra" | ||
| "github.com/spf13/pflag" | ||
| crashdCmd "github.com/vmware-tanzu/crash-diagnostics/cmd" | ||
| "k8s.io/client-go/tools/clientcmd" | ||
|
|
||
| "github.com/vmware-tanzu/velero/pkg/client" | ||
| "github.com/vmware-tanzu/velero/pkg/cmd" | ||
| ) | ||
|
|
||
| //go:embed cshd-scripts/velero.cshd | ||
| var scriptBytes []byte | ||
|
|
||
| type option struct { | ||
| // workdir for crashd will be $baseDir/velero-debug | ||
| baseDir string | ||
| // the namespace where velero server is installed | ||
| namespace string | ||
| // the absolute path for the log bundle to be generated | ||
| outputPath string | ||
| // the absolute path for the kubeconfig file that will be read by crashd for calling K8S API | ||
| kubeconfigPath string | ||
| // optional, the name of the backup resource whose log will be packaged into the debug bundle | ||
| backup string | ||
| // optional, the name of the restore resource whose log will be packaged into the debug bundle | ||
| restore string | ||
| } | ||
|
|
||
| func (o *option) bindFlags(flags *pflag.FlagSet) { | ||
| flags.StringVar(&o.outputPath, "output", "", "The path of the bundle tarball, by default it's $HOME/bundle.tar.gz. Optional") | ||
| flags.StringVar(&o.backup, "backup", "", "The name of the backup resource whose log will be collected, no backup logs will be collected if it's not set. Optional") | ||
| flags.StringVar(&o.restore, "restore", "", "The name of the restore resource whose log will be collected, no restore logs will be collected if it's not set. Optional") | ||
| } | ||
|
|
||
| func (o *option) asCrashdArgs() string { | ||
| return fmt.Sprintf("output=%s,namespace=%s,basedir=%s,backup=%s,restore=%s,kubeconfig=%s", | ||
| o.outputPath, o.namespace, o.baseDir, o.backup, o.restore, o.kubeconfigPath) | ||
| } | ||
|
|
||
| func (o *option) complete(f client.Factory, fs *pflag.FlagSet) error { | ||
| if len(o.outputPath) > 0 { | ||
| absOutputPath, err := filepath.Abs(o.outputPath) | ||
| if err != nil { | ||
| return fmt.Errorf("invalid output path: %v", err) | ||
| } | ||
| o.outputPath = absOutputPath | ||
| } | ||
| tmpDir, err := ioutil.TempDir("", "crashd") | ||
| if err != nil { | ||
| return err | ||
| } | ||
| o.baseDir = tmpDir | ||
| o.namespace = f.Namespace() | ||
| kp := kubeconfig(fs) | ||
| o.kubeconfigPath, err = filepath.Abs(kp) | ||
| if err != nil { | ||
| return fmt.Errorf("invalid kubeconfig path: %s, %v", kp, err) | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| // NewCommand creates a cobra command. | ||
| func NewCommand(f client.Factory) *cobra.Command { | ||
| o := &option{} | ||
| c := &cobra.Command{ | ||
| Use: "debug", | ||
| Short: "Generate debug bundle", | ||
| Long: `Generate a tarball containing the logs of velero deployment, plugin logs, restic DaemonSet, | ||
| specs of BackupStorageLocations, PodVolumeBackups, PodVolumeRestores, and optionally the specs and logs of backup and restore.`, | ||
| Run: func(c *cobra.Command, args []string) { | ||
| defer func(opt *option) { | ||
| if len(o.baseDir) > 0 { | ||
| if err := os.RemoveAll(o.baseDir); err != nil { | ||
| fmt.Fprintf(os.Stderr, "Failed to remove temp dir: %s: %v\n", o.baseDir, err) | ||
| } | ||
| } | ||
| }(o) | ||
| flags := c.Flags() | ||
| err := o.complete(f, flags) | ||
| cmd.CheckError(err) | ||
| err2 := runCrashd(o.asCrashdArgs()) | ||
| cmd.CheckError(err2) | ||
| }, | ||
| } | ||
| o.bindFlags(c.Flags()) | ||
| return c | ||
| } | ||
|
|
||
| func runCrashd(argString string) error { | ||
| bak := os.Args | ||
| defer func() { os.Args = bak }() | ||
| f, err := ioutil.TempFile("", "velero*.cshd") | ||
| if err != nil { | ||
| return err | ||
| } | ||
| defer func() { | ||
| if err := os.Remove(f.Name()); err != nil { | ||
| fmt.Fprintf(os.Stderr, "Failed to remove the temp file: %s, %v\n", f.Name(), err) | ||
| } | ||
| }() | ||
| _, err2 := f.Write(scriptBytes) | ||
| if err2 != nil { | ||
| return err2 | ||
| } | ||
| os.Args = []string{"", "run", "--debug", f.Name(), "--args", fmt.Sprintf("%s", argString)} | ||
| return crashdCmd.Run() | ||
| } | ||
|
|
||
| func kubeconfig(fs *pflag.FlagSet) string { | ||
| pathOpt := clientcmd.NewDefaultPathOptions() | ||
| kubeconfig, _ := fs.GetString("kubeconfig") | ||
| if len(kubeconfig) > 0 { | ||
| pathOpt.LoadingRules.ExplicitPath = kubeconfig | ||
| } | ||
| return pathOpt.GetDefaultFilename() | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters