-
Notifications
You must be signed in to change notification settings - Fork 303
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Browse files
Browse the repository at this point in the history
…agent-to-redactor Add cli command to redact secrets and redact secrets from Pipelines Secrets
- Loading branch information
Showing
20 changed files
with
773 additions
and
283 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,176 @@ | ||
| package clicommand | ||
|
|
||
| import ( | ||
| "bufio" | ||
| "context" | ||
| "encoding/json" | ||
| "errors" | ||
| "fmt" | ||
| "io" | ||
| "os" | ||
| "slices" | ||
| "strings" | ||
|
|
||
| "github.com/buildkite/agent/v3/jobapi" | ||
| "github.com/buildkite/agent/v3/logger" | ||
| "github.com/urfave/cli" | ||
| ) | ||
|
|
||
| // Note: if you add a new format string, make sure to add it to `secretsFormats` | ||
| // and update the usage string in LogRedactCommand | ||
| const ( | ||
| FormatStringJSON = "json" | ||
| FormatStringNone = "none" | ||
| // TODO: we should parse .env files | ||
| // TODO: we should parse ssh private keys. The format is in https://datatracker.ietf.org/doc/html/rfc7468 | ||
| ) | ||
|
|
||
| var ( | ||
| secretsFormats = []string{FormatStringJSON, FormatStringNone} | ||
|
|
||
| errSecretParse = errors.New("failed to parse secrets") | ||
| errSecretRedact = errors.New("failed to redact secrets") | ||
| errUnknownFormat = errors.New("unknown format") | ||
| ) | ||
|
|
||
| type RedactorAddConfig struct { | ||
| File string `cli:"arg:0"` | ||
| Format string `cli:"format"` | ||
|
|
||
| // Global flags | ||
| Debug bool `cli:"debug"` | ||
| LogLevel string `cli:"log-level"` | ||
| NoColor bool `cli:"no-color"` | ||
| Experiments []string `cli:"experiment" normalize:"list"` | ||
| Profile string `cli:"profile"` | ||
|
|
||
| // API config | ||
| DebugHTTP bool `cli:"debug-http"` | ||
| AgentAccessToken string `cli:"agent-access-token" validate:"required"` | ||
| Endpoint string `cli:"endpoint" validate:"required"` | ||
| NoHTTP2 bool `cli:"no-http2"` | ||
| } | ||
|
|
||
| var RedactorAddCommand = cli.Command{ | ||
| Name: "add", | ||
| Usage: "Add values to redact from a job's log output", | ||
| Description: "This may be used to parse a file for values to redact from a running job's log output. If you dynamically fetch secrets during a job, it is recommended that you use this command to ensure they will be redacted from subsequent logs. Secrects fetched with the builtin ′secret get′ command do not require the use of this command, they will be redacted automatically.", | ||
| Flags: []cli.Flag{ | ||
| cli.StringFlag{ | ||
| Name: "format", | ||
| Usage: fmt.Sprintf( | ||
| "The format for the input, one of: %s. ′none′ will add the entire input as a to the redactor, save for leading and trailing whitespace, ′json′ will parse it a string valued JSON Object, where each value of each key will be added to the redactor.", | ||
| secretsFormats, | ||
| ), | ||
| EnvVar: "BUILDKITE_AGENT_REDACT_ADD_FORMAT", | ||
| Value: FormatStringNone, | ||
| }, | ||
|
|
||
| // API Flags | ||
| AgentAccessTokenFlag, | ||
| EndpointFlag, | ||
| NoHTTP2Flag, | ||
| DebugHTTPFlag, | ||
|
|
||
| // Global flags | ||
| NoColorFlag, | ||
| DebugFlag, | ||
| LogLevelFlag, | ||
| ExperimentsFlag, | ||
| ProfileFlag, | ||
| }, | ||
| Action: func(c *cli.Context) error { | ||
| ctx := context.Background() | ||
| ctx, cfg, l, _, done := setupLoggerAndConfig[RedactorAddConfig](ctx, c) | ||
| defer done() | ||
|
|
||
| if !slices.Contains(secretsFormats, cfg.Format) { | ||
| return fmt.Errorf("invalid format: %s, must be one of %q", cfg.Format, secretsFormats) | ||
| } | ||
|
|
||
| fileName := "(stdin)" | ||
| // TODO: replace os.Stdin with c.App.Reader in cli v2+ | ||
| secretsReader := bufio.NewReader(os.Stdin) | ||
| if cfg.File != "" { | ||
| fileName = cfg.File | ||
|
|
||
| secretsFile, err := os.Open(fileName) | ||
| if err != nil { | ||
| return fmt.Errorf("failed to open file %s: %w", fileName, err) | ||
| } | ||
| defer secretsFile.Close() | ||
|
|
||
| secretsReader = bufio.NewReader(secretsFile) | ||
| } | ||
|
|
||
| l.Info("Reading secrets from %s for redaction", fileName) | ||
|
|
||
| secrets, err := ParseSecrets(l, cfg, secretsReader) | ||
| if err != nil { | ||
| if cfg.Debug { | ||
| return err | ||
| } | ||
| return errSecretParse | ||
| } | ||
|
|
||
| client, err := jobapi.NewDefaultClient(ctx) | ||
| if err != nil { | ||
| return fmt.Errorf("failed to create Job API client: %w", err) | ||
| } | ||
|
|
||
| if err := AddToRedactor(ctx, l, client, secrets...); err != nil { | ||
| if cfg.Debug { | ||
| return err | ||
| } | ||
| return errSecretRedact | ||
| } | ||
|
|
||
| return nil | ||
| }, | ||
| } | ||
|
|
||
| func ParseSecrets( | ||
| l logger.Logger, | ||
| cfg RedactorAddConfig, | ||
| secretsReader io.Reader, | ||
| ) ([]string, error) { | ||
| switch cfg.Format { | ||
| case FormatStringJSON: | ||
| secrets := &map[string]string{} | ||
| if err := json.NewDecoder(secretsReader).Decode(&secrets); err != nil { | ||
| return nil, fmt.Errorf("failed to parse as string valued JSON: %w", err) | ||
| } | ||
|
|
||
| parsedSecrets := make([]string, 0, len(*secrets)) | ||
| for _, secret := range *secrets { | ||
| parsedSecrets = append(parsedSecrets, secret) | ||
| } | ||
|
|
||
| return parsedSecrets, nil | ||
|
|
||
| case FormatStringNone: | ||
| readSecret, err := io.ReadAll(secretsReader) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("failed to read secret: %w", err) | ||
| } | ||
|
|
||
| return []string{strings.TrimSpace(string(readSecret))}, nil | ||
|
|
||
| default: | ||
| return nil, fmt.Errorf("%s: %w", cfg.Format, errUnknownFormat) | ||
| } | ||
| } | ||
|
|
||
| func AddToRedactor( | ||
| ctx context.Context, | ||
| l logger.Logger, | ||
| client *jobapi.Client, | ||
| secrets ...string, | ||
| ) error { | ||
| for _, secret := range secrets { | ||
| if _, err := client.RedactionCreate(ctx, secret); err != nil { | ||
| return fmt.Errorf("failed to add secret to the redactor: %w", err) | ||
| } | ||
| } | ||
| return nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,58 @@ | ||
| package clicommand_test | ||
|
|
||
| import ( | ||
| "slices" | ||
| "strings" | ||
| "testing" | ||
|
|
||
| "github.com/buildkite/agent/v3/clicommand" | ||
| "github.com/buildkite/agent/v3/logger" | ||
| "gotest.tools/v3/assert" | ||
| ) | ||
|
|
||
| func TestParseSecrets(t *testing.T) { | ||
| t.Parallel() | ||
|
|
||
| for _, tc := range []struct { | ||
| name string | ||
| inputData string | ||
| formatString string | ||
| expectedSecrets []string | ||
| errorTextContains string | ||
| }{ | ||
| { | ||
| name: "json", | ||
| inputData: `{"hello": "world", "password": "hunter2"}`, | ||
| formatString: clicommand.FormatStringJSON, | ||
| expectedSecrets: []string{"world", "hunter2"}, | ||
| }, | ||
| { | ||
| name: "plaintext", | ||
| inputData: "hunter2\n", | ||
| formatString: clicommand.FormatStringNone, | ||
| expectedSecrets: []string{"hunter2"}, | ||
| }, | ||
| { | ||
| name: "invalid_json", | ||
| inputData: `{"hello": 1, "password": "hunter2"}`, | ||
| formatString: clicommand.FormatStringJSON, | ||
| errorTextContains: "failed to parse as string valued JSON", | ||
| }, | ||
| } { | ||
| t.Run(tc.name, func(t *testing.T) { | ||
| t.Parallel() | ||
|
|
||
| input := strings.NewReader(tc.inputData) | ||
| secrets, err := clicommand.ParseSecrets(logger.Discard, clicommand.RedactorAddConfig{Format: tc.formatString}, input) | ||
| if tc.errorTextContains != "" { | ||
| assert.ErrorContains(t, err, tc.errorTextContains) | ||
| return | ||
| } | ||
| assert.NilError(t, err) | ||
|
|
||
| slices.Sort(secrets) | ||
| slices.Sort(tc.expectedSecrets) | ||
| assert.DeepEqual(t, secrets, tc.expectedSecrets) | ||
| }) | ||
| } | ||
| } |
Oops, something went wrong.