From 2587f3425ddb055517444055bcae73cb69510399 Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Wed, 5 Feb 2025 11:27:42 +0100 Subject: [PATCH] feat(lambda-promtail): Improve relabel configuration parsing and testing (#16100) --- tools/lambda-promtail/lambda-promtail/main.go | 14 +- .../lambda-promtail/relabel.go | 123 ++++++++++++++++++ .../lambda-promtail/relabel_test.go | 121 +++++++++++++++++ 3 files changed, 250 insertions(+), 8 deletions(-) create mode 100644 tools/lambda-promtail/lambda-promtail/relabel.go create mode 100644 tools/lambda-promtail/lambda-promtail/relabel_test.go diff --git a/tools/lambda-promtail/lambda-promtail/main.go b/tools/lambda-promtail/lambda-promtail/main.go index c462773869458..9c27491462f89 100644 --- a/tools/lambda-promtail/lambda-promtail/main.go +++ b/tools/lambda-promtail/lambda-promtail/main.go @@ -103,19 +103,17 @@ func setupArguments() { batchSize, _ = strconv.Atoi(batch) } - print := os.Getenv("PRINT_LOG_LINE") printLogLine = true - if strings.EqualFold(print, "false") { + if strings.EqualFold(os.Getenv("PRINT_LOG_LINE"), "false") { printLogLine = false } s3Clients = make(map[string]*s3.Client) - // Parse relabel configs from environment variable - if relabelConfigsRaw := os.Getenv("RELABEL_CONFIGS"); relabelConfigsRaw != "" { - if err := json.Unmarshal([]byte(relabelConfigsRaw), &relabelConfigs); err != nil { - panic(fmt.Errorf("failed to parse RELABEL_CONFIGS: %v", err)) - } + promConfigs, err := parseRelabelConfigs(os.Getenv("RELABEL_CONFIGS")) + if err != nil { + panic(err) } + relabelConfigs = promConfigs } func parseExtraLabels(extraLabelsRaw string, omitPrefix bool) (model.LabelSet, error) { @@ -131,7 +129,7 @@ func parseExtraLabels(extraLabelsRaw string, omitPrefix bool) (model.LabelSet, e } if len(extraLabelsSplit)%2 != 0 { - return nil, fmt.Errorf(invalidExtraLabelsError) + return nil, errors.New(invalidExtraLabelsError) } for i := 0; i < len(extraLabelsSplit); i += 2 { extractedLabels[model.LabelName(prefix+extraLabelsSplit[i])] = model.LabelValue(extraLabelsSplit[i+1]) diff --git a/tools/lambda-promtail/lambda-promtail/relabel.go b/tools/lambda-promtail/lambda-promtail/relabel.go new file mode 100644 index 0000000000000..7368ca4c6820f --- /dev/null +++ b/tools/lambda-promtail/lambda-promtail/relabel.go @@ -0,0 +1,123 @@ +package main + +import ( + "encoding/json" + "fmt" + + "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/model/relabel" +) + +// copy and modification of github.com/prometheus/prometheus/model/relabel/relabel.go +// reason: the custom types in github.com/prometheus/prometheus/model/relabel/relabel.go are difficult to unmarshal +type RelabelConfig struct { + // A list of labels from which values are taken and concatenated + // with the configured separator in order. + SourceLabels []string `json:"source_labels,omitempty"` + // Separator is the string between concatenated values from the source labels. + Separator string `json:"separator,omitempty"` + // Regex against which the concatenation is matched. + Regex string `json:"regex,omitempty"` + // Modulus to take of the hash of concatenated values from the source labels. + Modulus uint64 `json:"modulus,omitempty"` + // TargetLabel is the label to which the resulting string is written in a replacement. + // Regexp interpolation is allowed for the replace action. + TargetLabel string `json:"target_label,omitempty"` + // Replacement is the regex replacement pattern to be used. + Replacement string `json:"replacement,omitempty"` + // Action is the action to be performed for the relabeling. + Action string `json:"action,omitempty"` +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +func (rc *RelabelConfig) UnmarshalJSON(data []byte) error { + *rc = RelabelConfig{ + Action: string(relabel.Replace), + Separator: ";", + Regex: "(.*)", + Replacement: "$1", + } + type plain RelabelConfig + if err := json.Unmarshal(data, (*plain)(rc)); err != nil { + return err + } + return nil +} + +// ToPrometheusConfig converts our JSON-friendly RelabelConfig to the Prometheus RelabelConfig +func (rc *RelabelConfig) ToPrometheusConfig() (*relabel.Config, error) { + var regex relabel.Regexp + if rc.Regex != "" { + var err error + regex, err = relabel.NewRegexp(rc.Regex) + if err != nil { + return nil, fmt.Errorf("invalid regex %q: %w", rc.Regex, err) + } + } else { + regex = relabel.DefaultRelabelConfig.Regex + } + + action := relabel.Action(rc.Action) + if rc.Action == "" { + action = relabel.DefaultRelabelConfig.Action + } + + separator := rc.Separator + if separator == "" { + separator = relabel.DefaultRelabelConfig.Separator + } + + replacement := rc.Replacement + if replacement == "" { + replacement = relabel.DefaultRelabelConfig.Replacement + } + + sourceLabels := make(model.LabelNames, 0, len(rc.SourceLabels)) + for _, l := range rc.SourceLabels { + sourceLabels = append(sourceLabels, model.LabelName(l)) + } + + cfg := &relabel.Config{ + SourceLabels: sourceLabels, + Separator: separator, + Regex: regex, + Modulus: rc.Modulus, + TargetLabel: rc.TargetLabel, + Replacement: replacement, + Action: action, + } + + if err := cfg.Validate(); err != nil { + return nil, fmt.Errorf("invalid relabel config: %w", err) + } + return cfg, nil +} + +func ToPrometheusConfigs(cfgs []*RelabelConfig) ([]*relabel.Config, error) { + promConfigs := make([]*relabel.Config, 0, len(cfgs)) + for _, cfg := range cfgs { + promCfg, err := cfg.ToPrometheusConfig() + if err != nil { + return nil, fmt.Errorf("invalid relabel config: %w", err) + } + promConfigs = append(promConfigs, promCfg) + } + return promConfigs, nil +} + +func parseRelabelConfigs(relabelConfigsRaw string) ([]*relabel.Config, error) { + if relabelConfigsRaw == "" { + return nil, nil + } + + var relabelConfigs []*RelabelConfig + + if err := json.Unmarshal([]byte(relabelConfigsRaw), &relabelConfigs); err != nil { + return nil, fmt.Errorf("failed to parse RELABEL_CONFIGS: %v", err) + } + promConfigs, err := ToPrometheusConfigs(relabelConfigs) + if err != nil { + return nil, fmt.Errorf("failed to parse RELABEL_CONFIGS: %v", err) + } + return promConfigs, nil +} diff --git a/tools/lambda-promtail/lambda-promtail/relabel_test.go b/tools/lambda-promtail/lambda-promtail/relabel_test.go new file mode 100644 index 0000000000000..cb4cbebe45921 --- /dev/null +++ b/tools/lambda-promtail/lambda-promtail/relabel_test.go @@ -0,0 +1,121 @@ +package main + +import ( + "testing" + + "github.com/prometheus/prometheus/model/relabel" + "github.com/stretchr/testify/require" + + "github.com/grafana/regexp" +) + +func TestParseRelabelConfigs(t *testing.T) { + tests := []struct { + name string + input string + want []*relabel.Config + wantErr bool + }{ + { + name: "empty input", + input: "", + want: nil, + wantErr: false, + }, + { + name: "default config", + input: `[{"target_label": "new_label"}]`, + want: []*relabel.Config{ + { + TargetLabel: "new_label", + Action: relabel.Replace, + Regex: relabel.Regexp{Regexp: regexp.MustCompile("(.*)")}, + Replacement: "$1", + }, + }, + wantErr: false, + }, + { + name: "invalid JSON", + input: "invalid json", + wantErr: true, + }, + { + name: "valid single config", + input: `[{ + "source_labels": ["__name__"], + "regex": "my_metric_.*", + "target_label": "new_label", + "replacement": "foo", + "action": "replace" + }]`, + wantErr: false, + }, + { + name: "invalid regex", + input: `[{ + "source_labels": ["__name__"], + "regex": "[[invalid regex", + "target_label": "new_label", + "action": "replace" + }]`, + wantErr: true, + }, + { + name: "multiple valid configs", + input: `[ + { + "source_labels": ["__name__"], + "regex": "my_metric_.*", + "target_label": "new_label", + "replacement": "foo", + "action": "replace" + }, + { + "source_labels": ["label1", "label2"], + "separator": ";", + "regex": "val1;val2", + "target_label": "combined", + "action": "replace" + } + ]`, + wantErr: false, + }, + { + name: "invalid action", + input: `[{ + "source_labels": ["__name__"], + "regex": "my_metric_.*", + "target_label": "new_label", + "action": "invalid_action" + }]`, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := parseRelabelConfigs(tt.input) + if tt.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + + if tt.input == "" { + require.Nil(t, got) + return + } + + require.NotNil(t, got) + // For valid configs, verify they can be used for relabeling + // This implicitly tests that the conversion was successful + if len(got) > 0 { + for _, cfg := range got { + require.NotNil(t, cfg) + require.NotEmpty(t, cfg.Action) + } + } + }) + } +}