diff --git a/clicommand/agent_start.go b/clicommand/agent_start.go index 5d8ac0bed1..e53ec7b069 100644 --- a/clicommand/agent_start.go +++ b/clicommand/agent_start.go @@ -645,7 +645,7 @@ var AgentStartCommand = cli.Command{ }, cli.StringFlag{ Name: "signing-jwks-key-id", - Usage: "The JWKS key ID to use when signing the pipeline. If ommitted, and the signing JWKS conatins only one key, that key will be used.", + Usage: "The JWKS key ID to use when signing the pipeline. If omitted, and the signing JWKS contains only one key, that key will be used.", EnvVar: "BUILDKITE_AGENT_SIGNING_JWKS_KEY_ID", }, cli.StringFlag{ @@ -1082,7 +1082,7 @@ var AgentStartCommand = cli.Command{ Priority: cfg.Priority, ScriptEvalEnabled: !cfg.NoCommandEval, Tags: tags, - // We only want this agent to be ingored in Buildkite + // We only want this agent to be ignored in Buildkite // dispatches if it's being booted to acquire a // specific job. IgnoreInDispatches: cfg.AcquireJob != "", diff --git a/clicommand/commands.go b/clicommand/commands.go index bbdfb517bc..6c1825bfbb 100644 --- a/clicommand/commands.go +++ b/clicommand/commands.go @@ -3,8 +3,12 @@ package clicommand import "github.com/urfave/cli" var BuildkiteAgentCommands = []cli.Command{ - AcknowledgementsCommand, + // These commands are special. The have a different lifecycle to the others AgentStartCommand, + BootstrapCommand, + + // These are in alphabetical order + AcknowledgementsCommand, AnnotateCommand, { Name: "annotation", @@ -13,13 +17,6 @@ var BuildkiteAgentCommands = []cli.Command{ AnnotationRemoveCommand, }, }, - { - Name: "secret", - Usage: "Get a secret", - Subcommands: []cli.Command{ - SecretGetCommand, - }, - }, { Name: "artifact", Usage: "Upload/download artifacts from Buildkite jobs", @@ -30,7 +27,6 @@ var BuildkiteAgentCommands = []cli.Command{ ArtifactShasumCommand, }, }, - GitCredentialsHelperCommand, { Name: "env", Usage: "Process environment subcommands", @@ -41,6 +37,7 @@ var BuildkiteAgentCommands = []cli.Command{ EnvUnsetCommand, }, }, + GitCredentialsHelperCommand, { Name: "lock", Usage: "Process lock subcommands", @@ -52,6 +49,13 @@ var BuildkiteAgentCommands = []cli.Command{ LockReleaseCommand, }, }, + { + Name: "redactor", + Usage: "Redact sensitive information from logs", + Subcommands: []cli.Command{ + RedactorAddCommand, + }, + }, { Name: "meta-data", Usage: "Get/set data from Buildkite jobs", @@ -76,6 +80,13 @@ var BuildkiteAgentCommands = []cli.Command{ PipelineUploadCommand, }, }, + { + Name: "secret", + Usage: "Interact with Pipelines Secrets", + Subcommands: []cli.Command{ + SecretGetCommand, + }, + }, { Name: "step", Usage: "Get or update an attribute of a build step", @@ -84,7 +95,6 @@ var BuildkiteAgentCommands = []cli.Command{ StepUpdateCommand, }, }, - BootstrapCommand, { Name: "tool", Usage: "Utility commands, intended for users and operators of the agent to run directly on their machines, and not as part of a Buildkite job", diff --git a/clicommand/config_completeness_test.go b/clicommand/config_completeness_test.go index 847cd10996..2ca01fa293 100644 --- a/clicommand/config_completeness_test.go +++ b/clicommand/config_completeness_test.go @@ -39,6 +39,7 @@ var commandConfigPairs = []configCommandPair{ {Config: MetaDataSetConfig{}, Command: MetaDataSetCommand}, {Config: OIDCTokenConfig{}, Command: OIDCRequestTokenCommand}, {Config: PipelineUploadConfig{}, Command: PipelineUploadCommand}, + {Config: RedactorAddConfig{}, Command: RedactorAddCommand}, {Config: SecretGetConfig{}, Command: SecretGetCommand}, {Config: StepGetConfig{}, Command: StepGetCommand}, {Config: StepUpdateConfig{}, Command: StepUpdateCommand}, diff --git a/clicommand/redactor_add.go b/clicommand/redactor_add.go new file mode 100644 index 0000000000..ff164264ad --- /dev/null +++ b/clicommand/redactor_add.go @@ -0,0 +1,176 @@ +package clicommand + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "slices" + "strings" + + "github.com/buildkite/agent/v3/jobapi" + "github.com/buildkite/agent/v3/logger" + "github.com/urfave/cli" +) + +// Note: if you add a new format string, make sure to add it to `secretsFormats` +// and update the usage string in LogRedactCommand +const ( + FormatStringJSON = "json" + FormatStringNone = "none" + // TODO: we should parse .env files + // TODO: we should parse ssh private keys. The format is in https://datatracker.ietf.org/doc/html/rfc7468 +) + +var ( + secretsFormats = []string{FormatStringJSON, FormatStringNone} + + errSecretParse = errors.New("failed to parse secrets") + errSecretRedact = errors.New("failed to redact secrets") + errUnknownFormat = errors.New("unknown format") +) + +type RedactorAddConfig struct { + File string `cli:"arg:0"` + Format string `cli:"format"` + + // Global flags + Debug bool `cli:"debug"` + LogLevel string `cli:"log-level"` + NoColor bool `cli:"no-color"` + Experiments []string `cli:"experiment" normalize:"list"` + Profile string `cli:"profile"` + + // API config + DebugHTTP bool `cli:"debug-http"` + AgentAccessToken string `cli:"agent-access-token" validate:"required"` + Endpoint string `cli:"endpoint" validate:"required"` + NoHTTP2 bool `cli:"no-http2"` +} + +var RedactorAddCommand = cli.Command{ + Name: "add", + Usage: "Add values to redact from a job's log output", + Description: "This may be used to parse a file for values to redact from a running job's log output. If you dynamically fetch secrets during a job, it is recommended that you use this command to ensure they will be redacted from subsequent logs. Secrects fetched with the builtin ′secret get′ command do not require the use of this command, they will be redacted automatically.", + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "format", + Usage: fmt.Sprintf( + "The format for the input, one of: %s. ′none′ will add the entire input as a to the redactor, save for leading and trailing whitespace, ′json′ will parse it a string valued JSON Object, where each value of each key will be added to the redactor.", + secretsFormats, + ), + EnvVar: "BUILDKITE_AGENT_REDACT_ADD_FORMAT", + Value: FormatStringNone, + }, + + // API Flags + AgentAccessTokenFlag, + EndpointFlag, + NoHTTP2Flag, + DebugHTTPFlag, + + // Global flags + NoColorFlag, + DebugFlag, + LogLevelFlag, + ExperimentsFlag, + ProfileFlag, + }, + Action: func(c *cli.Context) error { + ctx := context.Background() + ctx, cfg, l, _, done := setupLoggerAndConfig[RedactorAddConfig](ctx, c) + defer done() + + if !slices.Contains(secretsFormats, cfg.Format) { + return fmt.Errorf("invalid format: %s, must be one of %q", cfg.Format, secretsFormats) + } + + fileName := "(stdin)" + // TODO: replace os.Stdin with c.App.Reader in cli v2+ + secretsReader := bufio.NewReader(os.Stdin) + if cfg.File != "" { + fileName = cfg.File + + secretsFile, err := os.Open(fileName) + if err != nil { + return fmt.Errorf("failed to open file %s: %w", fileName, err) + } + defer secretsFile.Close() + + secretsReader = bufio.NewReader(secretsFile) + } + + l.Info("Reading secrets from %s for redaction", fileName) + + secrets, err := ParseSecrets(l, cfg, secretsReader) + if err != nil { + if cfg.Debug { + return err + } + return errSecretParse + } + + client, err := jobapi.NewDefaultClient(ctx) + if err != nil { + return fmt.Errorf("failed to create Job API client: %w", err) + } + + if err := AddToRedactor(ctx, l, client, secrets...); err != nil { + if cfg.Debug { + return err + } + return errSecretRedact + } + + return nil + }, +} + +func ParseSecrets( + l logger.Logger, + cfg RedactorAddConfig, + secretsReader io.Reader, +) ([]string, error) { + switch cfg.Format { + case FormatStringJSON: + secrets := &map[string]string{} + if err := json.NewDecoder(secretsReader).Decode(&secrets); err != nil { + return nil, fmt.Errorf("failed to parse as string valued JSON: %w", err) + } + + parsedSecrets := make([]string, 0, len(*secrets)) + for _, secret := range *secrets { + parsedSecrets = append(parsedSecrets, secret) + } + + return parsedSecrets, nil + + case FormatStringNone: + readSecret, err := io.ReadAll(secretsReader) + if err != nil { + return nil, fmt.Errorf("failed to read secret: %w", err) + } + + return []string{strings.TrimSpace(string(readSecret))}, nil + + default: + return nil, fmt.Errorf("%s: %w", cfg.Format, errUnknownFormat) + } +} + +func AddToRedactor( + ctx context.Context, + l logger.Logger, + client *jobapi.Client, + secrets ...string, +) error { + for _, secret := range secrets { + if _, err := client.RedactionCreate(ctx, secret); err != nil { + return fmt.Errorf("failed to add secret to the redactor: %w", err) + } + } + return nil +} diff --git a/clicommand/redactor_add_test.go b/clicommand/redactor_add_test.go new file mode 100644 index 0000000000..b1dda5664b --- /dev/null +++ b/clicommand/redactor_add_test.go @@ -0,0 +1,58 @@ +package clicommand_test + +import ( + "slices" + "strings" + "testing" + + "github.com/buildkite/agent/v3/clicommand" + "github.com/buildkite/agent/v3/logger" + "gotest.tools/v3/assert" +) + +func TestParseSecrets(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + inputData string + formatString string + expectedSecrets []string + errorTextContains string + }{ + { + name: "json", + inputData: `{"hello": "world", "password": "hunter2"}`, + formatString: clicommand.FormatStringJSON, + expectedSecrets: []string{"world", "hunter2"}, + }, + { + name: "plaintext", + inputData: "hunter2\n", + formatString: clicommand.FormatStringNone, + expectedSecrets: []string{"hunter2"}, + }, + { + name: "invalid_json", + inputData: `{"hello": 1, "password": "hunter2"}`, + formatString: clicommand.FormatStringJSON, + errorTextContains: "failed to parse as string valued JSON", + }, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + input := strings.NewReader(tc.inputData) + secrets, err := clicommand.ParseSecrets(logger.Discard, clicommand.RedactorAddConfig{Format: tc.formatString}, input) + if tc.errorTextContains != "" { + assert.ErrorContains(t, err, tc.errorTextContains) + return + } + assert.NilError(t, err) + + slices.Sort(secrets) + slices.Sort(tc.expectedSecrets) + assert.DeepEqual(t, secrets, tc.expectedSecrets) + }) + } +} diff --git a/clicommand/secret_get.go b/clicommand/secret_get.go index 5bed308af9..1b2cb523b8 100644 --- a/clicommand/secret_get.go +++ b/clicommand/secret_get.go @@ -2,16 +2,17 @@ package clicommand import ( "context" - "errors" "fmt" - "os" "github.com/buildkite/agent/v3/api" + "github.com/buildkite/agent/v3/jobapi" "github.com/urfave/cli" ) type SecretGetConfig struct { - Key string `cli:"arg:0"` + Key string `cli:"arg:0"` + Job string `cli:"job" validate:"required"` + SkipRedaction bool `cli:"skip-redaction"` // Global flags Debug bool `cli:"debug"` @@ -27,13 +28,22 @@ type SecretGetConfig struct { NoHTTP2 bool `cli:"no-http2"` } -var errJobIDNotSet = errors.New("BUILDKITE_JOB_ID not set") - var SecretGetCommand = cli.Command{ Name: "get", Usage: "Get a secret by its key", - Description: "Get a secret by key from Buildkite and print it to stdout.", + Description: "Get a secret by key from Buildkite Pipelines Secrets and print it to stdout.", Flags: []cli.Flag{ + cli.StringFlag{ + Name: "job", + Usage: "Which job should should the secret be for", + EnvVar: "BUILDKITE_JOB_ID", + }, + cli.BoolFlag{ + Name: "skip-redaction", + Usage: "Skip redacting the retrieved secret from the logs. Then, the command will print the secret to the Job's logs if called directly.", + EnvVar: "BUILDKITE_AGENT_SECRET_GET_SKIP_SECRET_REDACTION", + }, + // API Flags AgentAccessTokenFlag, EndpointFlag, @@ -52,16 +62,24 @@ var SecretGetCommand = cli.Command{ ctx, cfg, l, _, done := setupLoggerAndConfig[SecretGetConfig](ctx, c) defer done() - client := api.NewClient(l, loadAPIClientConfig(cfg, "AgentAccessToken")) - - jobID := os.Getenv("BUILDKITE_JOB_ID") - if jobID == "" { - return NewExitError(1, errJobIDNotSet) + agentClient := api.NewClient(l, loadAPIClientConfig(cfg, "AgentAccessToken")) + secret, _, err := agentClient.GetSecret(ctx, &api.GetSecretRequest{Key: cfg.Key, JobID: cfg.Job}) + if err != nil { + return err } - secret, _, err := client.GetSecret(ctx, &api.GetSecretRequest{Key: cfg.Key, JobID: jobID}) + jobClient, err := jobapi.NewDefaultClient(ctx) if err != nil { - return NewExitError(1, err) + return fmt.Errorf("failed to create Job API client: %w", err) + } + + if !cfg.SkipRedaction { + if err := AddToRedactor(ctx, l, jobClient, secret.Value); err != nil { + if cfg.Debug { + return err + } + return errSecretRedact + } } _, err = fmt.Fprintln(c.App.Writer, secret.Value) diff --git a/internal/job/api.go b/internal/job/api.go index 084292ce4f..fb1e024b83 100644 --- a/internal/job/api.go +++ b/internal/job/api.go @@ -29,7 +29,7 @@ We'll continue to run your job, but you won't be able to use the Job API`) if e.ExecutorConfig.Debug { jobAPIOpts = append(jobAPIOpts, jobapi.WithDebug()) } - srv, token, err := jobapi.NewServer(e.shell.Logger, socketPath, e.shell.Env, jobAPIOpts...) + srv, token, err := jobapi.NewServer(e.shell.Logger, socketPath, e.shell.Env, e.redactors, jobAPIOpts...) if err != nil { return cleanup, fmt.Errorf("creating job API server: %v", err) } diff --git a/internal/job/executor.go b/internal/job/executor.go index d2d367bb83..461097c20d 100644 --- a/internal/job/executor.go +++ b/internal/job/executor.go @@ -58,6 +58,10 @@ type Executor struct { // A channel to track cancellation cancelCh chan struct{} + + // redactors for the job logs. The will be populated with values both from environment variable and through the Job API. + // In order for the latter to happen, a reference is passed into the the Job API server as well + redactors *replacer.Mux } // New returns a new executor instance @@ -65,6 +69,7 @@ func New(conf ExecutorConfig) *Executor { return &Executor{ ExecutorConfig: conf, cancelCh: make(chan struct{}), + redactors: replacer.NewMux(), } } @@ -97,10 +102,14 @@ func (e *Executor) Run(ctx context.Context) (exitCode int) { return 1 } defer func() { - kubernetesClient.Exit(exitCode) + _ = kubernetesClient.Exit(exitCode) }() } + // setup the redactors here once and for the life of the executor + // they will be flushed at the end of each hook + e.setupRedactors() + var err error span, ctx, stopper := e.startTracing(ctx) defer stopper() @@ -417,8 +426,7 @@ func logOpenedHookInfo(l shell.Logger, debug bool, hookName, path string) { } func (e *Executor) runWrappedShellScriptHook(ctx context.Context, hookName string, hookCfg HookConfig) error { - redactors := e.setupRedactors() - defer redactors.Flush() + defer e.redactors.Flush() script, err := hook.NewWrapper(hook.WithPath(hookCfg.Path)) if err != nil { @@ -502,13 +510,13 @@ func (e *Executor) runWrappedShellScriptHook(ctx context.Context, hookName strin } else { // Hook exited successfully (and not early!) We have an environment and // wd change we can apply to our subsequent phases - e.applyEnvironmentChanges(changes, redactors) + e.applyEnvironmentChanges(changes) } return nil } -func (e *Executor) applyEnvironmentChanges(changes hook.EnvChanges, redactors replacer.Mux) { +func (e *Executor) applyEnvironmentChanges(changes hook.EnvChanges) { if afterWd, err := changes.GetAfterWd(); err == nil { if afterWd != e.shell.Getwd() { _ = e.shell.Chdir(afterWd) @@ -523,7 +531,7 @@ func (e *Executor) applyEnvironmentChanges(changes hook.EnvChanges, redactors re e.shell.Env.Apply(changes.Diff) // reset output redactors based on new environment variable values - redactors.Reset(redact.Values(e.shell, e.ExecutorConfig.RedactedVars, e.shell.Env.Dump())) + e.redactors.Add(redact.Values(e.shell, e.ExecutorConfig.RedactedVars, e.shell.Env.Dump())...) // First, let see any of the environment variables are supposed // to change the job configuration at run time. @@ -878,6 +886,8 @@ func (e *Executor) CommandPhase(ctx context.Context) (hookErr error, commandErr // defaultCommandPhase is executed if there is no global or plugin command hook func (e *Executor) defaultCommandPhase(ctx context.Context) error { + defer e.redactors.Flush() + spanName := e.implementationSpecificSpanName("default command hook", "hook.execute") span, ctx := tracetools.StartSpanFromContext(ctx, spanName, e.ExecutorConfig.TracingBackend) var err error @@ -996,9 +1006,6 @@ func (e *Executor) defaultCommandPhase(ctx context.Context) error { return err } - redactors := e.setupRedactors() - defer redactors.Flush() - var cmd []string cmd = append(cmd, shell...) cmd = append(cmd, cmdToExec) @@ -1071,45 +1078,32 @@ func (e *Executor) writeBatchScript(cmd string) (string, error) { return scriptFile.Name(), nil } -// Check for ignored env variables from the job runner. Some -// env (for example, BUILDKITE_BUILD_PATH) can only be set from config or by hooks. -// If these env are set at a pipeline level, we rewrite them to BUILDKITE_X_BUILD_PATH -// and warn on them here so that users know what is going on -func (e *Executor) ignoredEnv() []string { - var ignored []string - for _, env := range os.Environ() { - if strings.HasPrefix(env, "BUILDKITE_X_") { - ignored = append(ignored, fmt.Sprintf("BUILDKITE_%s", - strings.TrimPrefix(env, "BUILDKITE_X_"))) - } - } - return ignored -} - -// setupRedactors wraps shell output and logging in Redactor if any redaction +// setupRedactors wraps shell output and logging in Redactors if any redaction // is necessary based on RedactedVars configuration and the existence of -// matching environment vars. -// redactor.Mux (possibly empty) is returned so the caller can `defer redactor.Flush()` -func (e *Executor) setupRedactors() replacer.Mux { +// matching environment vars. It will store the redactors in the Executor so +// that they may be updated when the environment changes. +// +// The returned method will remove the redactors from the Executor and Flush them. +func (e *Executor) setupRedactors() { valuesToRedact := redact.Values(e.shell, e.ExecutorConfig.RedactedVars, e.shell.Env.Dump()) if len(valuesToRedact) == 0 { - return nil + return } if e.Debug { e.shell.Commentf("Enabling output redaction for values from environment variables matching: %v", e.ExecutorConfig.RedactedVars) } - var mux replacer.Mux - // If the shell Writer is already a Replacer, reset the values to redact. if rdc, ok := e.shell.Writer.(*replacer.Replacer); ok { - rdc.Reset(valuesToRedact) - mux = append(mux, rdc) + // There may have been some redactees in the redactor already, so we + // propagate them to any new redactor. + valuesToRedact = append(valuesToRedact, rdc.Needles()...) + e.redactors.Append(rdc) } else { rdc := replacer.New(e.shell.Writer, valuesToRedact, redact.Redact) e.shell.Writer = rdc - mux = append(mux, rdc) + e.redactors.Append(rdc) } // If the shell.Logger is already a redacted WriterLogger, reset the values to redact. @@ -1124,15 +1118,17 @@ func (e *Executor) setupRedactors() replacer.Mux { } } if rdc := shellLoggerRedactor; rdc != nil { - rdc.Reset(valuesToRedact) - mux = append(mux, rdc) + // There may have been some redactees in the redactor already, so we + // propagate them to any new redactor. + valuesToRedact = append(valuesToRedact, rdc.Needles()...) + e.redactors.Append(rdc) } else if shellWriterLogger != nil { rdc := replacer.New(e.shell.Writer, valuesToRedact, redact.Redact) shellWriterLogger.Writer = rdc - mux = append(mux, rdc) + e.redactors.Append(rdc) } - return mux + e.redactors.Add(valuesToRedact...) } func (e *Executor) startKubernetesClient(ctx context.Context, kubernetesClient *kubernetes.Client) error { diff --git a/internal/olfactor/olfactor.go b/internal/olfactor/olfactor.go index 2f51ec809c..b6aa4a5ff1 100644 --- a/internal/olfactor/olfactor.go +++ b/internal/olfactor/olfactor.go @@ -16,7 +16,7 @@ type Olfactor struct { } // New returns an io.Writer and an Olfactor. Writes to the writer will be -// forwarded to `dst` and the returned Olfactor will recored whether the +// forwarded to `dst` and the returned Olfactor will record whether the // elements of `smells` have been written to the io.Writer. // // If a smell is the empty string, we consider it to have been smelt, even if diff --git a/internal/replacer/mux.go b/internal/replacer/mux.go new file mode 100644 index 0000000000..28b7fa4305 --- /dev/null +++ b/internal/replacer/mux.go @@ -0,0 +1,47 @@ +package replacer + +import ( + "errors" +) + +// Mux contains multiple replacers +type Mux struct { + underlying []*Replacer +} + +// NewMux returns a new mux with the given replacers. +func NewMux(rs ...*Replacer) *Mux { + m := &Mux{ + underlying: make([]*Replacer, 0, len(rs)), + } + m.underlying = append(m.underlying, rs...) + return m +} + +// Reset resets all replacers with new needles (secrets). +func (m *Mux) Reset(needles []string) { + for _, r := range m.underlying { + r.Reset(needles) + } +} + +// Add adds needles to all replacers. +func (m *Mux) Add(needles ...string) { + for _, r := range m.underlying { + r.Add(needles...) + } +} + +// Append adds a replacer to the Mux. +func (m *Mux) Append(r *Replacer) { + m.underlying = append(m.underlying, r) +} + +// Flush flushes all replacers. +func (m *Mux) Flush() error { + errs := make([]error, 0, len(m.underlying)) + for _, r := range m.underlying { + errs = append(errs, r.Flush()) + } + return errors.Join(errs...) +} diff --git a/internal/replacer/replacer.go b/internal/replacer/replacer.go index 037fc8fead..e8e968c7ad 100644 --- a/internal/replacer/replacer.go +++ b/internal/replacer/replacer.go @@ -2,18 +2,19 @@ package replacer import ( - "errors" "fmt" "io" "sync" ) -// Replacer is a straightforward streaming string replacer and replacer, -// suitable for detecting or redacting secrets in a stream. +type unit struct{} + +// Replacer is a straightforward streaming string replacer suitable for +// detecting or redacting secrets in a stream. // // The algorithm is intended to be easier to maintain than certain // high-performance multi-string search algorithms, and also geared towards -// ensuring secrets don't escape (for instance, by matching overlaps), at the +// ensuring strings don't escape (for instance, by matching overlaps), at the // expense of ultimate efficiency. type Replacer struct { // The replacement callback. @@ -23,7 +24,7 @@ type Replacer struct { // organised by first byte. // Why first byte? Because looking up needles by the first byte is a lot // faster than _filtering_ all the needles by first byte. - needlesByFirstByte [256][]string + needlesByFirstByte [256]map[string]unit // For synchronising writes. Each write can touch everything below. mu sync.Mutex @@ -34,8 +35,8 @@ type Replacer struct { // Intermediate buffer to account for partially-written data. buf []byte - // Current redaction partialMatches - if we have begun redacting a potential - // secret there will be at least one of these. + // Current redaction partialMatches - if we have begun matching a potential + // needle there will be at least one of these. // nextMatches is the next set of partialMatches. // Write alternates between these two, rather than creating a new slice to // hold the next set of matches for every byte of the input. @@ -79,7 +80,7 @@ func New(dst io.Writer, needles []string, replacement func([]byte) []byte) *Repl return r } -// Write searches the stream for needles (strings, secrets, ...), calls the +// Write searches the stream for needles (e.g. strings, secrets, ...), calls the // replacement callback to obtain any replacements, and forwards the output to // the destination writer. func (r *Replacer) Write(b []byte) (int, error) { @@ -99,7 +100,7 @@ func (r *Replacer) Write(b []byte) (int, error) { // matches. // // Step 2 is complicated by the fact that each Write could contain a partial - // secret at the start or the end. So a buffer is needed to hold onto any + // needle at the start or the end. So a buffer is needed to hold onto any // incomplete matches (in case they _don't_ match), as well as some extra // state (r.partialMatches) for tracking where we are in each incomplete // match. @@ -144,7 +145,7 @@ func (r *Replacer) Write(b []byte) (int, error) { } // Start matching something? - for _, s := range r.needlesByFirstByte[c] { + for s, _ := range r.needlesByFirstByte[c] { if len(s) == 1 { // A pathological case; in practice we don't redact secrets // smaller than RedactLengthMin. @@ -287,11 +288,34 @@ func (r *Replacer) flushUpTo(limit int) error { return nil } -// Reset replaces the secrets to redact with a new set of secrets. It is not +// Size returns the number of needles +func (r *Replacer) Size() int { + sum := 0 + for _, n := range r.needlesByFirstByte { + sum += len(n) + } + return sum +} + +// Needle returns the current needles +func (r *Replacer) Needles() []string { + r.mu.Lock() + defer r.mu.Unlock() + + needles := make([]string, 0, r.Size()) + for _, m := range r.needlesByFirstByte { + for n := range m { + needles = append(needles, n) + } + } + return needles +} + +// Reset removes all current needes and sets new set of needles. It is not // necessary to Flush beforehand, but: -// - any previous secrets which have begun matching will continue matching +// - any previous needles which have begun matching will continue matching // (until they reach a terminal state), and -// - any new secrets will not be compared against existing buffer content, +// - any new needles will not be compared against existing buffer content, // only data passed to Write calls after Reset. func (r *Replacer) Reset(needles []string) { r.mu.Lock() @@ -300,11 +324,33 @@ func (r *Replacer) Reset(needles []string) { for i := range r.needlesByFirstByte { r.needlesByFirstByte[i] = nil } + + r.unsafeAdd(needles) +} + +// Add adds more needles to be matched by the replacer. It is not necessary to +// Flush beforehand, but: +// - any previous strings which have begun matching will continue matching +// (until they reach a terminal state), and +// - any new strings will not be compared against existing buffer content, +// only data passed to Write calls after Add. +func (r *Replacer) Add(needles ...string) { + r.mu.Lock() + defer r.mu.Unlock() + + r.unsafeAdd(needles) +} + +func (r *Replacer) unsafeAdd(needles []string) { for _, s := range needles { if len(s) == 0 { continue } - r.needlesByFirstByte[s[0]] = append(r.needlesByFirstByte[s[0]], s) + if r.needlesByFirstByte[s[0]] == nil { + r.needlesByFirstByte[s[0]] = map[string]unit{s: {}} + continue + } + r.needlesByFirstByte[s[0]][s] = unit{} } } @@ -372,27 +418,3 @@ func mergeOverlaps(rs []subrange) []subrange { copy(rs, rs[j:]) return rs[:rem] } - -// Mux contains multiple replacers -type Mux []*Replacer - -// Flush flushes all replacers. -func (mux Mux) Flush() error { - var errs []error - for _, r := range mux { - if err := r.Flush(); err != nil { - errs = append(errs, err) - } - } - if len(errs) != 0 { - return errors.Join(errs...) - } - return nil -} - -// Reset resets all replacers with new needles (secrets). -func (mux Mux) Reset(needles []string) { - for _, r := range mux { - r.Reset(needles) - } -} diff --git a/internal/replacer/replacer_test.go b/internal/replacer/replacer_test.go index 7889949c21..bc731bcbaf 100644 --- a/internal/replacer/replacer_test.go +++ b/internal/replacer/replacer_test.go @@ -4,12 +4,14 @@ import ( "bytes" "fmt" "io" + "slices" "strings" "testing" "github.com/buildkite/agent/v3/internal/redact" "github.com/buildkite/agent/v3/internal/replacer" "github.com/google/go-cmp/cmp" + "gotest.tools/v3/assert" ) const lipsum = "Lorem ipsum dolor sit amet" @@ -233,6 +235,36 @@ func TestReplacerMultiLine(t *testing.T) { } } +func TestAddingNeedles(t *testing.T) { + t.Parallel() + + needles := []string{"secret1111", "secret2222"} + afterAddExpectedNeedles := []string{"secret1111", "secret2222", "pre-secret3333"} + + var buf strings.Builder + replacer := replacer.New(&buf, needles, redact.Redact) + actualNeedles := replacer.Needles() + + slices.Sort(needles) + slices.Sort(actualNeedles) + assert.DeepEqual(t, actualNeedles, needles) + + _, err := replacer.Write([]byte("redact secret1111 and secret2222 but not pre-secret3333\n")) + assert.NilError(t, err) + + replacer.Add("pre-secret3333") + actualNeedles = replacer.Needles() + slices.Sort(actualNeedles) + slices.Sort(afterAddExpectedNeedles) + assert.DeepEqual(t, actualNeedles, afterAddExpectedNeedles) + + _, err = replacer.Write([]byte("now redact secret1111, secret2222, and pre-secret3333\n")) + assert.NilError(t, err) + assert.NilError(t, replacer.Flush()) + + assert.Equal(t, buf.String(), "redact [REDACTED] and [REDACTED] but not pre-secret3333\nnow redact [REDACTED], [REDACTED], and [REDACTED]\n") +} + func BenchmarkReplacer(b *testing.B) { b.ResetTimer() r := replacer.New(io.Discard, bigLipsumSecrets, redact.Redact) diff --git a/internal/socket/client.go b/internal/socket/client.go index e5557b184a..dc19235c4b 100644 --- a/internal/socket/client.go +++ b/internal/socket/client.go @@ -101,7 +101,8 @@ func (c *Client) Do(ctx context.Context, method, url string, req, resp any) erro defer hresp.Body.Close() dec := json.NewDecoder(hresp.Body) - if hresp.StatusCode != http.StatusOK { + switch hresp.StatusCode / 100 { + case 4, 5: var er ErrorResponse if err := dec.Decode(&er); err != nil { return fmt.Errorf("decoding error response: %w", err) diff --git a/jobapi/client.go b/jobapi/client.go index 0b7efcd61f..df65fc83c9 100644 --- a/jobapi/client.go +++ b/jobapi/client.go @@ -3,12 +3,16 @@ package jobapi import ( "context" "errors" + "net/http" "os" "github.com/buildkite/agent/v3/internal/socket" ) -const envURL = "http://job/api/current-job/v0/env" +const ( + envURL = "http://job/api/current-job/v0/env" + redactionsURL = "http://job/api/current-job/v0/redactions" +) var errNoSocketEnv = errors.New("BUILDKITE_AGENT_JOB_API_SOCKET empty or undefined") @@ -81,3 +85,15 @@ func (c *Client) EnvDelete(ctx context.Context, del []string) (deleted []string, resp.Normalize() return resp.Deleted, nil } + +// RedactionCreate creates a redaction in the job executor. +func (c *Client) RedactionCreate(ctx context.Context, text string) (string, error) { + req := RedactionCreateRequest{ + Redact: text, + } + var resp RedactionCreateResponse + if err := c.client.Do(ctx, http.MethodPost, redactionsURL, &req, &resp); err != nil { + return "", err + } + return resp.Redacted, nil +} diff --git a/jobapi/env.go b/jobapi/env.go new file mode 100644 index 0000000000..6346c40cec --- /dev/null +++ b/jobapi/env.go @@ -0,0 +1,148 @@ +package jobapi + +import ( + "encoding/json" + "fmt" + "net/http" + + "github.com/buildkite/agent/v3/agent" + "github.com/buildkite/agent/v3/internal/socket" + "golang.org/x/exp/maps" +) + +func (s *Server) getEnv(w http.ResponseWriter, _ *http.Request) { + s.mtx.RLock() + normalizedEnv := s.environ.Dump() + s.mtx.RUnlock() + + resp := EnvGetResponse{Env: normalizedEnv} + w.WriteHeader(http.StatusOK) + if err := json.NewEncoder(w).Encode(resp); err != nil { + s.Logger.Errorf("Job API: couldn't encode or write response: %v", err) + } +} + +func (s *Server) patchEnv(w http.ResponseWriter, r *http.Request) { + var req EnvUpdateRequestPayload + err := json.NewDecoder(r.Body).Decode(&req) + defer r.Body.Close() + if err != nil { + if err := socket.WriteError(w, fmt.Errorf("failed to decode request body: %w", err), http.StatusBadRequest); err != nil { + s.Logger.Errorf("Job API: couldn't write error: %v", err) + } + return + } + + added := make([]string, 0, len(req.Env)) + updated := make([]string, 0, len(req.Env)) + protected := checkProtected(maps.Keys(req.Env)) + + if len(protected) > 0 { + err := socket.WriteError( + w, + fmt.Sprintf("the following environment variables are protected, and cannot be modified: % v", protected), + http.StatusUnprocessableEntity, + ) + if err != nil { + s.Logger.Errorf("Job API: couldn't write error: %v", err) + } + return + } + + nils := make([]string, 0, len(req.Env)) + + for k, v := range req.Env { + if v == nil { + nils = append(nils, k) + } + } + + if len(nils) > 0 { + err := socket.WriteError( + w, + fmt.Sprintf("removing environment variables (ie setting them to null) is not permitted on this endpoint. The following keys were set to null: % v", nils), + http.StatusUnprocessableEntity, + ) + if err != nil { + s.Logger.Errorf("Job API: couldn't write error: %v", err) + } + return + } + + s.mtx.Lock() + for k, v := range req.Env { + if _, ok := s.environ.Get(k); ok { + updated = append(updated, k) + } else { + added = append(added, k) + } + s.environ.Set(k, *v) + } + s.mtx.Unlock() + + resp := EnvUpdateResponse{ + Added: added, + Updated: updated, + } + + resp.Normalize() + + w.WriteHeader(http.StatusOK) + if err := json.NewEncoder(w).Encode(resp); err != nil { + s.Logger.Errorf("Job API: couldn't encode or write response: %v", err) + } +} + +func (s *Server) deleteEnv(w http.ResponseWriter, r *http.Request) { + var req EnvDeleteRequest + err := json.NewDecoder(r.Body).Decode(&req) + defer r.Body.Close() + if err != nil { + err := socket.WriteError(w, fmt.Errorf("failed to decode request body: %w", err), http.StatusBadRequest) + if err != nil { + s.Logger.Errorf("Job API: couldn't write error: %v", err) + } + return + } + + protected := checkProtected(req.Keys) + if len(protected) > 0 { + err := socket.WriteError( + w, + fmt.Sprintf("the following environment variables are protected, and cannot be modified: % v", protected), + http.StatusUnprocessableEntity, + ) + if err != nil { + s.Logger.Errorf("Job API: couldn't write error: %v", err) + } + return + } + + s.mtx.Lock() + deleted := make([]string, 0, len(req.Keys)) + for _, k := range req.Keys { + if _, ok := s.environ.Get(k); ok { + deleted = append(deleted, k) + s.environ.Remove(k) + } + } + s.mtx.Unlock() + + resp := EnvDeleteResponse{Deleted: deleted} + resp.Normalize() + + w.WriteHeader(http.StatusOK) + if err := json.NewEncoder(w).Encode(resp); err != nil { + s.Logger.Errorf("Job API: couldn't encode or write response: %v", err) + } +} + +func checkProtected(candidates []string) []string { + protected := make([]string, 0, len(candidates)) + for _, c := range candidates { + if _, ok := agent.ProtectedEnv[c]; ok { + protected = append(protected, c) + } + } + return protected +} diff --git a/jobapi/payloads.go b/jobapi/payloads.go index c6e51bb25e..e69c139c05 100644 --- a/jobapi/payloads.go +++ b/jobapi/payloads.go @@ -48,3 +48,13 @@ type EnvDeleteResponse struct { func (e EnvDeleteResponse) Normalize() { sort.Strings(e.Deleted) } + +// RedactionCreateRequest is the request body for the POST /redactions endpoint +type RedactionCreateRequest struct { + Redact string `json:"redact"` +} + +// RedactionCreateResponse is the response body for the POST /redactions endpoint +type RedactionCreateResponse struct { + Redacted string `json:"redacted"` +} diff --git a/jobapi/redactions.go b/jobapi/redactions.go new file mode 100644 index 0000000000..bca16436e2 --- /dev/null +++ b/jobapi/redactions.go @@ -0,0 +1,29 @@ +package jobapi + +import ( + "encoding/json" + "fmt" + "net/http" + + "github.com/buildkite/agent/v3/internal/socket" +) + +func (s *Server) createRedaction(w http.ResponseWriter, r *http.Request) { + payload := &RedactionCreateRequest{} + if err := json.NewDecoder(r.Body).Decode(payload); err != nil { + if err := socket.WriteError(w, fmt.Errorf("failed to decode request body: %w", err), http.StatusBadRequest); err != nil { + s.Logger.Errorf("Job API: couldn't write error: %v", err) + } + return + } + + s.mtx.Lock() + s.redactors.Add(payload.Redact) + s.mtx.Unlock() + + respBody := &RedactionCreateResponse{Redacted: payload.Redact} + w.WriteHeader(http.StatusCreated) + if err := json.NewEncoder(w).Encode(respBody); err != nil { + s.Logger.Errorf("Job API: couldn't write error: %v", err) + } +} diff --git a/jobapi/routes.go b/jobapi/routes.go index 134a9479dc..35e40ee766 100644 --- a/jobapi/routes.go +++ b/jobapi/routes.go @@ -1,15 +1,11 @@ package jobapi import ( - "encoding/json" - "fmt" "net/http" - "github.com/buildkite/agent/v3/agent" "github.com/buildkite/agent/v3/internal/socket" "github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5/middleware" - "golang.org/x/exp/maps" ) // router returns a chi router with the jobapi routes and appropriate middlewares mounted @@ -33,144 +29,9 @@ func (s *Server) router() chi.Router { r.Get("/env", s.getEnv) r.Patch("/env", s.patchEnv) r.Delete("/env", s.deleteEnv) + + r.Post("/redactions", s.createRedaction) }) return r } - -func (s *Server) getEnv(w http.ResponseWriter, _ *http.Request) { - s.mtx.RLock() - normalizedEnv := s.environ.Dump() - s.mtx.RUnlock() - - resp := EnvGetResponse{Env: normalizedEnv} - w.WriteHeader(http.StatusOK) - if err := json.NewEncoder(w).Encode(resp); err != nil { - s.Logger.Errorf("Job API: couldn't encode or write response: %v", err) - } -} - -func (s *Server) patchEnv(w http.ResponseWriter, r *http.Request) { - var req EnvUpdateRequestPayload - err := json.NewDecoder(r.Body).Decode(&req) - defer r.Body.Close() - if err != nil { - if err := socket.WriteError(w, fmt.Errorf("failed to decode request body: %w", err), http.StatusBadRequest); err != nil { - s.Logger.Errorf("Job API: couldn't write error: %v", err) - } - return - } - - added := make([]string, 0, len(req.Env)) - updated := make([]string, 0, len(req.Env)) - protected := checkProtected(maps.Keys(req.Env)) - - if len(protected) > 0 { - err := socket.WriteError( - w, - fmt.Sprintf("the following environment variables are protected, and cannot be modified: % v", protected), - http.StatusUnprocessableEntity, - ) - if err != nil { - s.Logger.Errorf("Job API: couldn't write error: %v", err) - } - return - } - - nils := make([]string, 0, len(req.Env)) - - for k, v := range req.Env { - if v == nil { - nils = append(nils, k) - } - } - - if len(nils) > 0 { - err := socket.WriteError( - w, - fmt.Sprintf("removing environment variables (ie setting them to null) is not permitted on this endpoint. The following keys were set to null: % v", nils), - http.StatusUnprocessableEntity, - ) - if err != nil { - s.Logger.Errorf("Job API: couldn't write error: %v", err) - } - return - } - - s.mtx.Lock() - for k, v := range req.Env { - if _, ok := s.environ.Get(k); ok { - updated = append(updated, k) - } else { - added = append(added, k) - } - s.environ.Set(k, *v) - } - s.mtx.Unlock() - - resp := EnvUpdateResponse{ - Added: added, - Updated: updated, - } - - resp.Normalize() - - w.WriteHeader(http.StatusOK) - if err := json.NewEncoder(w).Encode(resp); err != nil { - s.Logger.Errorf("Job API: couldn't encode or write response: %v", err) - } -} - -func (s *Server) deleteEnv(w http.ResponseWriter, r *http.Request) { - var req EnvDeleteRequest - err := json.NewDecoder(r.Body).Decode(&req) - defer r.Body.Close() - if err != nil { - err := socket.WriteError(w, fmt.Errorf("failed to decode request body: %w", err), http.StatusBadRequest) - if err != nil { - s.Logger.Errorf("Job API: couldn't write error: %v", err) - } - return - } - - protected := checkProtected(req.Keys) - if len(protected) > 0 { - err := socket.WriteError( - w, - fmt.Sprintf("the following environment variables are protected, and cannot be modified: % v", protected), - http.StatusUnprocessableEntity, - ) - if err != nil { - s.Logger.Errorf("Job API: couldn't write error: %v", err) - } - return - } - - s.mtx.Lock() - deleted := make([]string, 0, len(req.Keys)) - for _, k := range req.Keys { - if _, ok := s.environ.Get(k); ok { - deleted = append(deleted, k) - s.environ.Remove(k) - } - } - s.mtx.Unlock() - - resp := EnvDeleteResponse{Deleted: deleted} - resp.Normalize() - - w.WriteHeader(http.StatusOK) - if err := json.NewEncoder(w).Encode(resp); err != nil { - s.Logger.Errorf("Job API: couldn't encode or write response: %v", err) - } -} - -func checkProtected(candidates []string) []string { - protected := make([]string, 0, len(candidates)) - for _, c := range candidates { - if _, ok := agent.ProtectedEnv[c]; ok { - protected = append(protected, c) - } - } - return protected -} diff --git a/jobapi/server.go b/jobapi/server.go index f6814aba7e..b6efc03439 100644 --- a/jobapi/server.go +++ b/jobapi/server.go @@ -9,6 +9,7 @@ import ( "github.com/buildkite/agent/v3/env" "github.com/buildkite/agent/v3/internal/job/shell" + "github.com/buildkite/agent/v3/internal/replacer" "github.com/buildkite/agent/v3/internal/socket" ) @@ -29,8 +30,9 @@ type Server struct { Logger shell.Logger debug bool - mtx sync.RWMutex - environ *env.Environment + mtx sync.RWMutex + environ *env.Environment + redactors *replacer.Mux token string sockSvr *socket.Server @@ -43,6 +45,7 @@ func NewServer( logger shell.Logger, socketPath string, environ *env.Environment, + redactors *replacer.Mux, opts ...ServerOpts, ) (server *Server, token string, err error) { token, err = socket.GenerateToken(32) @@ -54,6 +57,7 @@ func NewServer( SocketPath: socketPath, Logger: logger, environ: environ, + redactors: redactors, token: token, } diff --git a/jobapi/server_test.go b/jobapi/server_test.go index 1b3281076b..01924e08ab 100644 --- a/jobapi/server_test.go +++ b/jobapi/server_test.go @@ -16,8 +16,11 @@ import ( "github.com/buildkite/agent/v3/env" "github.com/buildkite/agent/v3/internal/job/shell" + "github.com/buildkite/agent/v3/internal/redact" + "github.com/buildkite/agent/v3/internal/replacer" "github.com/buildkite/agent/v3/jobapi" "github.com/google/go-cmp/cmp" + "gotest.tools/v3/assert" ) func pt(s string) *string { @@ -32,13 +35,12 @@ func testEnviron() *env.Environment { return e } -func testServer(t *testing.T, e *env.Environment) (*jobapi.Server, string, error) { +func testServer(t *testing.T, e *env.Environment, mux *replacer.Mux) (*jobapi.Server, string, error) { sockName, err := jobapi.NewSocketPath(os.TempDir()) if err != nil { return nil, "", fmt.Errorf("creating socket path: %w", err) } - - return jobapi.NewServer(shell.TestingLogger{T: t}, sockName, e) + return jobapi.NewServer(shell.TestingLogger{T: t}, sockName, e, mux) } func testSocketClient(socketPath string) *http.Client { @@ -51,11 +53,46 @@ func testSocketClient(socketPath string) *http.Client { } } +func testAPI[Req, Resp any](t *testing.T, env *env.Environment, req *http.Request, client *http.Client, testCase apiTestCase[Req, Resp]) { + t.Helper() + + resp, err := client.Do(req) + if err != nil { + t.Fatalf("expected no error for client.Do(req) (got %v)", err) + } + + if resp.StatusCode != testCase.expectedStatus { + t.Fatalf("expected status code %d (got %d)", testCase.expectedStatus, resp.StatusCode) + } + + if testCase.expectedResponseBody != nil { + var got Resp + assert.NilError(t, json.NewDecoder(resp.Body).Decode(&got)) + if !cmp.Equal(testCase.expectedResponseBody, &got) { + t.Fatalf("\n\texpected response: % #v\n\tgot: % #v\n\tdiff = %s)", *testCase.expectedResponseBody, got, cmp.Diff(testCase.expectedResponseBody, &got)) + } + } + + if testCase.expectedError != nil { + var got jobapi.ErrorResponse + assert.NilError(t, json.NewDecoder(resp.Body).Decode(&got)) + if got.Error != testCase.expectedError.Error { + t.Fatalf("expected error %q (got %q)", testCase.expectedError.Error, got.Error) + } + } + + if testCase.expectedEnv != nil { + if !cmp.Equal(testCase.expectedEnv, env.Dump()) { + t.Fatalf("\n\texpected env: % #v\n\tgot: % #v\n\tdiff = %s)", testCase.expectedEnv, env, cmp.Diff(testCase.expectedEnv, env)) + } + } +} + func TestServerStartStop(t *testing.T) { t.Parallel() env := testEnviron() - srv, _, err := testServer(t, env) + srv, _, err := testServer(t, env, replacer.NewMux()) if err != nil { t.Fatalf("testServer(t, env) error = %v", err) } @@ -100,12 +137,14 @@ func TestServerStartStop(t *testing.T) { } func TestServerStartStop_WithPreExistingSocket(t *testing.T) { + t.Parallel() + if runtime.GOOS == "windows" { t.Skip("socket collision detection isn't support on windows. If the current go version is >1.23, it might be worth re-enabling this test, because hopefully the bug (https://github.com/golang/go/issues/33357) is fixed") } sockName := filepath.Join(os.TempDir(), "test-socket-collision.sock") - srv1, _, err := jobapi.NewServer(shell.TestingLogger{T: t}, sockName, env.New()) + srv1, _, err := jobapi.NewServer(shell.TestingLogger{T: t}, sockName, env.New(), replacer.NewMux()) if err != nil { t.Fatalf("expected initial server creation to succeed, got %v", err) } @@ -117,7 +156,7 @@ func TestServerStartStop_WithPreExistingSocket(t *testing.T) { defer srv1.Stop() expectedErr := fmt.Sprintf("creating socket server: file already exists at socket path %s", sockName) - _, _, err = jobapi.NewServer(shell.TestingLogger{T: t}, sockName, env.New()) + _, _, err = jobapi.NewServer(shell.TestingLogger{T: t}, sockName, env.New(), replacer.NewMux()) if err == nil { t.Fatalf("expected second server creation to fail with %s, got nil", expectedErr) } @@ -172,7 +211,7 @@ func TestDeleteEnv(t *testing.T) { t.Parallel() environ := testEnviron() - srv, token, err := testServer(t, environ) + srv, token, err := testServer(t, environ, replacer.NewMux()) if err != nil { t.Fatalf("creating server: %v", err) } @@ -268,7 +307,7 @@ func TestPatchEnv(t *testing.T) { t.Parallel() environ := testEnviron() - srv, token, err := testServer(t, environ) + srv, token, err := testServer(t, environ, replacer.NewMux()) if err != nil { t.Fatalf("creating server: %v", err) } @@ -310,7 +349,7 @@ func TestGetEnv(t *testing.T) { t.Parallel() env := testEnviron() - srv, token, err := testServer(t, env) + srv, token, err := testServer(t, env, replacer.NewMux()) if err != nil { t.Fatalf("creating server: %v", err) } @@ -361,35 +400,57 @@ func TestGetEnv(t *testing.T) { }) } -func testAPI[Req, Resp any](t *testing.T, env *env.Environment, req *http.Request, client *http.Client, testCase apiTestCase[Req, Resp]) { - resp, err := client.Do(req) - if err != nil { - t.Fatalf("expected no error for client.Do(req) (got %v)", err) - } +func TestCreateRedaction(t *testing.T) { + t.Parallel() - if resp.StatusCode != testCase.expectedStatus { - t.Fatalf("expected status code %d (got %d)", testCase.expectedStatus, resp.StatusCode) - } + const ( + alreadyRedacted = "Guayaquil" + toRedact = "Quito" + ) - if testCase.expectedResponseBody != nil { - var got Resp - json.NewDecoder(resp.Body).Decode(&got) - if !cmp.Equal(testCase.expectedResponseBody, &got) { - t.Fatalf("\n\texpected response: % #v\n\tgot: % #v\n\tdiff = %s)", *testCase.expectedResponseBody, got, cmp.Diff(testCase.expectedResponseBody, &got)) - } - } + writeBuf := &bytes.Buffer{} + rdc := replacer.New(writeBuf, []string{alreadyRedacted}, redact.Redact) + mux := replacer.NewMux(rdc) - if testCase.expectedError != nil { - var got jobapi.ErrorResponse - json.NewDecoder(resp.Body).Decode(&got) - if got.Error != testCase.expectedError.Error { - t.Fatalf("expected error %q (got %q)", testCase.expectedError.Error, got.Error) - } - } + env := testEnviron() + srv, token, err := testServer(t, env, mux) + assert.NilError(t, err) - if testCase.expectedEnv != nil { - if !cmp.Equal(testCase.expectedEnv, env.Dump()) { - t.Fatalf("\n\texpected env: % #v\n\tgot: % #v\n\tdiff = %s)", testCase.expectedEnv, env, cmp.Diff(testCase.expectedEnv, env)) - } + // write some stuff that won't be redacted + _, err = rdc.Write([]byte("Go from Guayaquil, until you get to Quito.\n")) + assert.NilError(t, err) + + assert.NilError(t, srv.Start()) + t.Cleanup(func() { + assert.NilError(t, srv.Stop()) + }) + + client := testSocketClient(srv.SocketPath) + + tc := apiTestCase[jobapi.RedactionCreateRequest, jobapi.RedactionCreateResponse]{ + expectedStatus: http.StatusCreated, + requestBody: &jobapi.RedactionCreateRequest{Redact: toRedact}, + expectedResponseBody: &jobapi.RedactionCreateResponse{Redacted: toRedact}, } + + buf := &bytes.Buffer{} + assert.NilError(t, json.NewEncoder(buf).Encode(tc.requestBody)) + + req, err := http.NewRequest(http.MethodPost, "http://job/api/current-job/v0/redactions", buf) + assert.NilError(t, err) + + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token)) + testAPI(t, env, req, client, tc) + + // now when we write it, it should be redacted + _, err = rdc.Write([]byte("From Quito, go back to Guayaquil.\n")) + assert.NilError(t, err) + + mux.Flush() + + assert.Equal( + t, + writeBuf.String(), + "Go from [REDACTED], until you get to Quito.\nFrom [REDACTED], go back to [REDACTED].\n", + ) }