diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 397070b9f254..cbf55f4bb9c2 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -248,6 +248,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add support for RFC3339 time zone offsets in JSON output. {pull}13227[13227] - Add autodetection mode for add_docker_metadata and enable it by default in included configuration files{pull}13374[13374] - Added `monitoring.cluster_uuid` setting to associate Beat data with specified ES cluster in Stack Monitoring UI. {pull}13182[13182] +- Add autodetection mode for add_kubernetes_metadata and enable it by default in included configuration files. {pull}13473[13473] + *Auditbeat* diff --git a/auditbeat/auditbeat.yml b/auditbeat/auditbeat.yml index d3db66e60d57..a914f510073c 100644 --- a/auditbeat/auditbeat.yml +++ b/auditbeat/auditbeat.yml @@ -149,6 +149,7 @@ processors: - add_host_metadata: ~ - add_cloud_metadata: ~ - add_docker_metadata: ~ + - add_kubernetes_metadata: ~ #================================ Logging ===================================== diff --git a/filebeat/filebeat.yml b/filebeat/filebeat.yml index a404af47c161..d02b4d161fbf 100644 --- a/filebeat/filebeat.yml +++ b/filebeat/filebeat.yml @@ -177,6 +177,7 @@ processors: - add_host_metadata: ~ - add_cloud_metadata: ~ - add_docker_metadata: ~ + - add_kubernetes_metadata: ~ #================================ Logging ===================================== diff --git a/journalbeat/journalbeat.yml b/journalbeat/journalbeat.yml index ee2ce4502f05..5e8aad8b4ad4 100644 --- a/journalbeat/journalbeat.yml +++ b/journalbeat/journalbeat.yml @@ -146,6 +146,7 @@ processors: - add_host_metadata: ~ - add_cloud_metadata: ~ - add_docker_metadata: ~ + - add_kubernetes_metadata: ~ #================================ Logging ===================================== diff --git a/libbeat/_meta/config.yml.tmpl b/libbeat/_meta/config.yml.tmpl index e687c0cc7679..9ef6a1c8adc7 100644 --- a/libbeat/_meta/config.yml.tmpl +++ b/libbeat/_meta/config.yml.tmpl @@ -94,6 +94,7 @@ processors: - add_host_metadata: ~ - add_cloud_metadata: ~ - add_docker_metadata: ~ + - add_kubernetes_metadata: ~ {{else}} processors: - add_observer_metadata: diff --git a/libbeat/processors/add_kubernetes_metadata/kubernetes.go b/libbeat/processors/add_kubernetes_metadata/kubernetes.go index ea635314b079..c9461c35bf99 100644 --- a/libbeat/processors/add_kubernetes_metadata/kubernetes.go +++ b/libbeat/processors/add_kubernetes_metadata/kubernetes.go @@ -21,6 +21,8 @@ import ( "fmt" "time" + k8sclient "k8s.io/client-go/kubernetes" + "github.com/elastic/beats/libbeat/beat" "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/common/kubernetes" @@ -33,10 +35,11 @@ const ( ) type kubernetesAnnotator struct { - watcher kubernetes.Watcher - indexers *Indexers - matchers *Matchers - cache *cache + watcher kubernetes.Watcher + indexers *Indexers + matchers *Matchers + cache *cache + kubernetesAvailable bool } func init() { @@ -51,6 +54,16 @@ func init() { Indexing.AddMatcher(FieldFormatMatcherName, NewFieldFormatMatcher) } +func isKubernetesAvailable(client k8sclient.Interface) bool { + server, err := client.Discovery().ServerVersion() + if err != nil { + logp.Info("%v: could not detect kubernetes env: %v", "add_kubernetes_metadata", err) + return false + } + logp.Info("%v: kubernetes env detected, with version: %v", "add_kubernetes_metadata", server) + return true +} + // New constructs a new add_kubernetes_metadata processor. func New(cfg *common.Config) (processors.Processor, error) { config := defaultKubernetesAnnotatorConfig() @@ -91,9 +104,25 @@ func New(cfg *common.Config) (processors.Processor, error) { return nil, fmt.Errorf("Can not initialize kubernetes plugin with zero matcher plugins") } + processor := &kubernetesAnnotator{ + indexers: indexers, + matchers: matchers, + cache: newCache(config.CleanupTimeout), + kubernetesAvailable: false, + } + client, err := kubernetes.GetKubernetesClient(config.KubeConfig) if err != nil { - return nil, err + if kubernetes.IsInCluster(config.KubeConfig) { + logp.Debug("kubernetes", "%v: could not create kubernetes client using in_cluster config", "add_kubernetes_metadata") + } else { + logp.Debug("kubernetes", "%v: could not create kubernetes client using config: %v", "add_kubernetes_metadata", config.KubeConfig) + } + return processor, nil + } + + if !isKubernetesAvailable(client) { + return processor, nil } config.Host = kubernetes.DiscoverKubernetesNode(config.Host, kubernetes.IsInCluster(config.KubeConfig), client) @@ -111,12 +140,8 @@ func New(cfg *common.Config) (processors.Processor, error) { return nil, err } - processor := &kubernetesAnnotator{ - watcher: watcher, - indexers: indexers, - matchers: matchers, - cache: newCache(config.CleanupTimeout), - } + processor.watcher = watcher + processor.kubernetesAvailable = true watcher.AddEventHandler(kubernetes.ResourceEventHandlerFuncs{ AddFunc: func(obj interface{}) { @@ -139,6 +164,9 @@ func New(cfg *common.Config) (processors.Processor, error) { } func (k *kubernetesAnnotator) Run(event *beat.Event) (*beat.Event, error) { + if !k.kubernetesAvailable { + return event, nil + } index := k.matchers.MetadataIndex(event.Fields) if index == "" { return event, nil diff --git a/libbeat/processors/add_kubernetes_metadata/kubernetes_test.go b/libbeat/processors/add_kubernetes_metadata/kubernetes_test.go index c3467b5c96f2..4da15db969b4 100644 --- a/libbeat/processors/add_kubernetes_metadata/kubernetes_test.go +++ b/libbeat/processors/add_kubernetes_metadata/kubernetes_test.go @@ -42,6 +42,7 @@ func TestAnnotatorDeepUpdate(t *testing.T) { matchers: &Matchers{ matchers: []Matcher{matcher}, }, + kubernetesAvailable: true, } processor.cache.set("foo", common.MapStr{ @@ -86,3 +87,42 @@ func TestAnnotatorDeepUpdate(t *testing.T) { }, }, event.Fields) } + +// Test metadata are not included in the event +func TestAnnotatorWithNoKubernetesAvailable(t *testing.T) { + cfg := common.MustNewConfigFrom(map[string]interface{}{ + "lookup_fields": []string{"kubernetes.pod.name"}, + }) + matcher, err := NewFieldMatcher(*cfg) + if err != nil { + t.Fatal(err) + } + + processor := kubernetesAnnotator{ + cache: newCache(10 * time.Second), + matchers: &Matchers{ + matchers: []Matcher{matcher}, + }, + kubernetesAvailable: false, + } + + intialEventMap := common.MapStr{ + "kubernetes": common.MapStr{ + "pod": common.MapStr{ + "name": "foo", + "id": "pod_id", + "metrics": common.MapStr{ + "a": 1, + "b": 2, + }, + }, + }, + } + + event, err := processor.Run(&beat.Event{ + Fields: intialEventMap.Clone(), + }) + assert.NoError(t, err) + + assert.Equal(t, intialEventMap, event.Fields) +} diff --git a/metricbeat/metricbeat.yml b/metricbeat/metricbeat.yml index 8fb745093b8b..6d2d0519b442 100644 --- a/metricbeat/metricbeat.yml +++ b/metricbeat/metricbeat.yml @@ -121,6 +121,7 @@ processors: - add_host_metadata: ~ - add_cloud_metadata: ~ - add_docker_metadata: ~ + - add_kubernetes_metadata: ~ #================================ Logging ===================================== diff --git a/packetbeat/packetbeat.yml b/packetbeat/packetbeat.yml index 0a306eaeb6bf..965e5c5975a6 100644 --- a/packetbeat/packetbeat.yml +++ b/packetbeat/packetbeat.yml @@ -203,6 +203,7 @@ processors: - add_host_metadata: ~ - add_cloud_metadata: ~ - add_docker_metadata: ~ + - add_kubernetes_metadata: ~ #================================ Logging ===================================== diff --git a/winlogbeat/winlogbeat.yml b/winlogbeat/winlogbeat.yml index 253c5d4131f1..7d900eea1a14 100644 --- a/winlogbeat/winlogbeat.yml +++ b/winlogbeat/winlogbeat.yml @@ -128,6 +128,7 @@ processors: - add_host_metadata: ~ - add_cloud_metadata: ~ - add_docker_metadata: ~ + - add_kubernetes_metadata: ~ #================================ Logging ===================================== diff --git a/x-pack/auditbeat/auditbeat.yml b/x-pack/auditbeat/auditbeat.yml index 814609baefe9..a4d0f54f1cb7 100644 --- a/x-pack/auditbeat/auditbeat.yml +++ b/x-pack/auditbeat/auditbeat.yml @@ -171,6 +171,7 @@ processors: - add_host_metadata: ~ - add_cloud_metadata: ~ - add_docker_metadata: ~ + - add_kubernetes_metadata: ~ #================================ Logging ===================================== diff --git a/x-pack/filebeat/filebeat.yml b/x-pack/filebeat/filebeat.yml index a404af47c161..d02b4d161fbf 100644 --- a/x-pack/filebeat/filebeat.yml +++ b/x-pack/filebeat/filebeat.yml @@ -177,6 +177,7 @@ processors: - add_host_metadata: ~ - add_cloud_metadata: ~ - add_docker_metadata: ~ + - add_kubernetes_metadata: ~ #================================ Logging ===================================== diff --git a/x-pack/functionbeat/functionbeat.yml b/x-pack/functionbeat/functionbeat.yml index 06bf3681dd87..8182bddeced6 100644 --- a/x-pack/functionbeat/functionbeat.yml +++ b/x-pack/functionbeat/functionbeat.yml @@ -345,6 +345,7 @@ processors: - add_host_metadata: ~ - add_cloud_metadata: ~ - add_docker_metadata: ~ + - add_kubernetes_metadata: ~ #================================ Logging ===================================== diff --git a/x-pack/metricbeat/metricbeat.yml b/x-pack/metricbeat/metricbeat.yml index 8fb745093b8b..6d2d0519b442 100644 --- a/x-pack/metricbeat/metricbeat.yml +++ b/x-pack/metricbeat/metricbeat.yml @@ -121,6 +121,7 @@ processors: - add_host_metadata: ~ - add_cloud_metadata: ~ - add_docker_metadata: ~ + - add_kubernetes_metadata: ~ #================================ Logging ===================================== diff --git a/x-pack/winlogbeat/winlogbeat.yml b/x-pack/winlogbeat/winlogbeat.yml index 35ddff7e9fa0..690ed27a9eac 100644 --- a/x-pack/winlogbeat/winlogbeat.yml +++ b/x-pack/winlogbeat/winlogbeat.yml @@ -140,6 +140,7 @@ processors: - add_host_metadata: ~ - add_cloud_metadata: ~ - add_docker_metadata: ~ + - add_kubernetes_metadata: ~ #================================ Logging =====================================