-
Notifications
You must be signed in to change notification settings - Fork 389
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
metrics: Refactor metrics label filter logic
Tetragon has a functionality to disable some of the high-cardinality metrics
labels. So far it worked like this:
1. Configure option.Config.MetricsLabelFilter as a set (map[string]interface{})
of enabled labels.
2. metrics.granularLabelFilter wraps enabled labels and configurable labels in
one global variable.
3. Define a metric using a "granular metric" wrapper, which uses
granularLabelFilter.
4. When the metric is registered, the inner Prometheus metric is registered
with the enabled labels only.
5. Then we update the metric like a regular Prometheus metric, using
WithLabelValues.
Refactor this logic so that now it works like this:
1. Configure MetricsLabelFilter similarly like before, but as map[string]bool
2. metrics.ProcessLabels struct implements metrics.FilteredLabels interface and
contains label values of configurable labels. Disabled labels are set to ""
when the struct is instantiated.
3. Define a metric using a "granular metric" wrapper, which now uses Go generics
to specify configurable labels (FilteredMetrics type).
4. The inner Prometheus metric is now registered with all labels.
5. Then we update the metric using a slightly different implementation of
WithLabelValues, which takes a generic FileredLabels struct and any
additional label values as strings.
This approach provides better types, so that it's not as easy for developers to
make a mistake and pass incorrect labels when updating a metric. Additionally,
it makes it easy to define a different set of configurable labels (another
FilteredMetrics type) and to implement dynamic metrics labels configuration (as
it doesn't require re-registering metrics when the filter changes).
Signed-off-by: Anna Kapuscinska <anna@isovalent.com>- Loading branch information
Showing
11 changed files
with
273 additions
and
264 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,49 @@ | ||
| // SPDX-License-Identifier: Apache-2.0 | ||
| // Copyright Authors of Tetragon | ||
|
|
||
| package metrics | ||
|
|
||
| import ( | ||
| "github.com/cilium/tetragon/pkg/option" | ||
| ) | ||
|
|
||
| type FilteredLabels interface { | ||
| Keys() []string | ||
| Values() []string | ||
| } | ||
|
|
||
| type ProcessLabels struct { | ||
| Namespace string | ||
| Workload string | ||
| Pod string | ||
| Binary string | ||
| } | ||
|
|
||
| func NewProcessLabels(namespace, workload, pod, binary string) *ProcessLabels { | ||
| if !option.Config.MetricsLabelFilter["namespace"] { | ||
| namespace = "" | ||
| } | ||
| if !option.Config.MetricsLabelFilter["workload"] { | ||
| workload = "" | ||
| } | ||
| if !option.Config.MetricsLabelFilter["pod"] { | ||
| pod = "" | ||
| } | ||
| if !option.Config.MetricsLabelFilter["binary"] { | ||
| binary = "" | ||
| } | ||
| return &ProcessLabels{ | ||
| Namespace: namespace, | ||
| Workload: workload, | ||
| Pod: pod, | ||
| Binary: binary, | ||
| } | ||
| } | ||
|
|
||
| func (l ProcessLabels) Keys() []string { | ||
| return []string{"namespace", "workload", "pod", "binary"} | ||
| } | ||
|
|
||
| func (l ProcessLabels) Values() []string { | ||
| return []string{l.Namespace, l.Workload, l.Pod, l.Binary} | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,37 @@ | ||
| // SPDX-License-Identifier: Apache-2.0 | ||
| // Copyright Authors of Tetragon | ||
|
|
||
| package metrics | ||
|
|
||
| import ( | ||
| "testing" | ||
|
|
||
| "github.com/stretchr/testify/assert" | ||
|
|
||
| "github.com/cilium/tetragon/pkg/metrics/consts" | ||
| "github.com/cilium/tetragon/pkg/option" | ||
| ) | ||
|
|
||
| func TestProcessLabels(t *testing.T) { | ||
| namespace := "test-namespace" | ||
| workload := "test-deployment" | ||
| pod := "test-deployment-d9jo2" | ||
| binary := "test-binary" | ||
|
|
||
| // check that all labels are enabled by default (this relies on other tests | ||
| // not changing option.Config.MetricsLabelFilter) | ||
| processLabels := NewProcessLabels(namespace, workload, pod, binary) | ||
| assert.Equal(t, processLabels.Values(), []string{namespace, workload, pod, binary}) | ||
|
|
||
| // enable only namespace and binary | ||
| option.Config.MetricsLabelFilter = map[string]bool{ | ||
| "namespace": true, | ||
| "binary": true, | ||
| } | ||
| // check that labels are filtered correctly | ||
| processLabels = NewProcessLabels(namespace, workload, pod, binary) | ||
| assert.Equal(t, processLabels.Values(), []string{namespace, "", "", binary}) | ||
|
|
||
| // clean up - reset the config back to the default | ||
| option.Config.MetricsLabelFilter = consts.DefaultProcessLabels | ||
| } |
Oops, something went wrong.