Skip to content

Commit

Permalink
Prepare docker autodiscover in 6.7 for ECS and dedotting (#10899)
Browse files Browse the repository at this point in the history 
Partial backport of #10898 to 6.7

Summary of backported changes:
    * ECS fields are added as autodiscover selectors
    * Labels dedotting added, disabled by default

Co-Authored-By: kaiyan-sheng <kaiyan.sheng@elastic.co>
Co-Authored-By: Nicolas Ruflin <spam@ruflin.com>
  • Loading branch information
@jsoriano @kaiyan-sheng @ruflin
3 people authored Feb 26, 2019
1 parent da41f72 commit 7dbf4cc
Show file tree
Hide file tree
Showing 6 changed files with 209 additions and 29 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.next.asciidoc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ https://github.com/elastic/beats/compare/v6.6.0...6.x[Check the HEAD diff]
- Do not panic when no tokenizer string is configured for a dissect processor. {issue}8895[8895]
- Fix stopping of modules started by kubernetes autodiscover. {pull}10476[10476]
- Fix a issue when remote and local configuration didn't match when fetching configuration from Central Management. {issue}10587[10587]
- Add ECS-like selectors and dedotting to docker autodiscover. {issue}10757[10757] {pull}10862[10862]

*Auditbeat*

Expand Down
2 changes: 2 additions & 0 deletions heartbeat/tests/system/test_autodiscovery.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,3 +62,5 @@ def test_docker(self):
matched = True

assert matched

self.assert_fields_are_documented(output[0])
2 changes: 2 additions & 0 deletions libbeat/autodiscover/providers/docker/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,12 +32,14 @@ type Config struct {
Builders []*common.Config `config:"builders"`
Appenders []*common.Config `config:"appenders"`
Templates template.MapperSettings `config:"templates"`
Dedot bool `config:"labels.dedot"`
}

func defaultConfig() *Config {
return &Config{
Host: "unix:///var/run/docker.sock",
Prefix: "co.elastic",
Dedot: false,
}
}

Expand Down
109 changes: 80 additions & 29 deletions libbeat/autodiscover/providers/docker/docker.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@
package docker

import (
"errors"

"github.com/gofrs/uuid"

"github.com/elastic/beats/libbeat/autodiscover"
Expand Down Expand Up @@ -119,41 +121,91 @@ func (d *Provider) Start() {
}()
}

func (d *Provider) emitContainer(event bus.Event, flag string) {
type dockerMetadata struct {
// Old selectors [Deprecated]
Docker common.MapStr

// New ECS-based selectors
Container common.MapStr

// Metadata used to enrich events, like ECS-based selectors but can
// have modifications like dedotting
Metadata common.MapStr
}

func (d *Provider) generateMetaDocker(event bus.Event) (*docker.Container, *dockerMetadata) {
container, ok := event["container"].(*docker.Container)
if !ok {
logp.Err("Couldn't get a container from watcher event")
return
logp.Error(errors.New("Couldn't get a container from watcher event"))
return nil, nil
}

var host string
if len(container.IPAddresses) > 0 {
host = container.IPAddresses[0]
}
// Don't dedot selectors, dedot only metadata used for events enrichment
labelMap := common.MapStr{}
metaLabelMap := common.MapStr{}
for k, v := range container.Labels {
safemapstr.Put(labelMap, k, v)
if d.config.Dedot {
label := common.DeDot(k)
metaLabelMap.Put(label, v)
} else {
safemapstr.Put(metaLabelMap, k, v)
}
}

meta := common.MapStr{
"container": common.MapStr{
"id": container.ID,
"name": container.Name,
"image": container.Image,
meta := &dockerMetadata{
Docker: common.MapStr{
"container": common.MapStr{
"id": container.ID,
"name": container.Name,
"image": container.Image,
"labels": labelMap,
},
},
Container: common.MapStr{
"id": container.ID,
"name": container.Name,
"image": common.MapStr{
"name": container.Image,
},
"labels": labelMap,
},
Metadata: common.MapStr{
"docker": common.MapStr{
"container": common.MapStr{
"id": container.ID,
"name": container.Name,
"image": container.Image,
"labels": metaLabelMap,
},
},
},
}

return container, meta
}

func (d *Provider) emitContainer(event bus.Event, flag string) {
container, meta := d.generateMetaDocker(event)
if container == nil || meta == nil {
return
}

var host string
if len(container.IPAddresses) > 0 {
host = container.IPAddresses[0]
}

// Without this check there would be overlapping configurations with and without ports.
if len(container.Ports) == 0 {
event := bus.Event{
"provider": d.uuid,
"id": container.ID,
flag: true,
"host": host,
"docker": meta,
"meta": common.MapStr{
"docker": meta,
},
"provider": d.uuid,
"id": container.ID,
flag: true,
"host": host,
"docker": meta.Docker,
"container": meta.Container,
"meta": meta.Metadata,
}

d.publish(event)
Expand All @@ -162,15 +214,14 @@ func (d *Provider) emitContainer(event bus.Event, flag string) {
// Emit container container and port information
for _, port := range container.Ports {
event := bus.Event{
"provider": d.uuid,
"id": container.ID,
flag: true,
"host": host,
"port": port.PrivatePort,
"docker": meta,
"meta": common.MapStr{
"docker": meta,
},
"provider": d.uuid,
"id": container.ID,
flag: true,
"host": host,
"port": port.PrivatePort,
"docker": meta.Docker,
"container": meta.Container,
"meta": meta.Metadata,
}

d.publish(event)
Expand Down
121 changes: 121 additions & 0 deletions libbeat/autodiscover/providers/docker/docker_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import (

"github.com/elastic/beats/libbeat/common"
"github.com/elastic/beats/libbeat/common/bus"
"github.com/elastic/beats/libbeat/common/docker"
)

func TestGenerateHints(t *testing.T) {
Expand Down Expand Up @@ -105,3 +106,123 @@ func getNestedAnnotations(in common.MapStr) common.MapStr {
}
return out
}

func TestGenerateMetaDockerNoDedot(t *testing.T) {
event := bus.Event{
"container": &docker.Container{
ID: "abc",
Name: "foobar",
Labels: map[string]string{
"do.not.include": "true",
"co.elastic.logs/disable": "true",
},
},
}

cfg := defaultConfig()
cfg.Dedot = false
p := Provider{
config: cfg,
}
_, meta := p.generateMetaDocker(event)
expectedMeta := &dockerMetadata{
Docker: common.MapStr{
"container": common.MapStr{
"id": "abc",
"name": "foobar",
"image": "",
"labels": common.MapStr{
"do": common.MapStr{"not": common.MapStr{"include": "true"}},
"co": common.MapStr{"elastic": common.MapStr{"logs/disable": "true"}},
},
},
},
Container: common.MapStr{
"id": "abc",
"name": "foobar",
"image": common.MapStr{
"name": "",
},
"labels": common.MapStr{
"do": common.MapStr{"not": common.MapStr{"include": "true"}},
"co": common.MapStr{"elastic": common.MapStr{"logs/disable": "true"}},
},
},
Metadata: common.MapStr{
"docker": common.MapStr{
"container": common.MapStr{
"id": "abc",
"name": "foobar",
"image": "",
"labels": common.MapStr{
"do": common.MapStr{"not": common.MapStr{"include": "true"}},
"co": common.MapStr{"elastic": common.MapStr{"logs/disable": "true"}},
},
},
},
},
}
assert.Equal(t, expectedMeta.Docker, meta.Docker)
assert.Equal(t, expectedMeta.Container, meta.Container)
assert.Equal(t, expectedMeta.Metadata, meta.Metadata)
}

func TestGenerateMetaDockerWithDedot(t *testing.T) {
event := bus.Event{
"container": &docker.Container{
ID: "abc",
Name: "foobar",
Labels: map[string]string{
"do.not.include": "true",
"co.elastic.logs/disable": "true",
},
},
}

cfg := defaultConfig()
cfg.Dedot = true
p := Provider{
config: cfg,
}
_, meta := p.generateMetaDocker(event)
expectedMeta := &dockerMetadata{
Docker: common.MapStr{
"container": common.MapStr{
"id": "abc",
"name": "foobar",
"image": "",
"labels": common.MapStr{
"do": common.MapStr{"not": common.MapStr{"include": "true"}},
"co": common.MapStr{"elastic": common.MapStr{"logs/disable": "true"}},
},
},
},
Container: common.MapStr{
"id": "abc",
"name": "foobar",
"image": common.MapStr{
"name": "",
},
"labels": common.MapStr{
"do": common.MapStr{"not": common.MapStr{"include": "true"}},
"co": common.MapStr{"elastic": common.MapStr{"logs/disable": "true"}},
},
},
Metadata: common.MapStr{
"docker": common.MapStr{
"container": common.MapStr{
"id": "abc",
"name": "foobar",
"image": "",
"labels": common.MapStr{
"do_not_include": "true",
"co_elastic_logs/disable": "true",
},
},
},
},
}
assert.Equal(t, expectedMeta.Docker, meta.Docker)
assert.Equal(t, expectedMeta.Container, meta.Container)
assert.Equal(t, expectedMeta.Metadata, meta.Metadata)
}
3 changes: 3 additions & 0 deletions metricbeat/tests/system/test_autodiscover.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ def test_docker(self):
assert output[0]['docker']['container']['image'] == 'memcached:latest'
assert output[0]['docker']['container']['labels'] == {}
assert 'name' in output[0]['docker']['container']
self.assert_fields_are_documented(output[0])

@unittest.skipIf(not INTEGRATION_TESTS or
os.getenv("TESTING_ENVIRONMENT") == "2x",
Expand Down Expand Up @@ -95,6 +96,7 @@ def test_docker_labels(self):
# Check metadata is added
assert output[0]['docker']['container']['image'] == 'memcached:latest'
assert 'name' in output[0]['docker']['container']
self.assert_fields_are_documented(output[0])

@unittest.skipIf(not INTEGRATION_TESTS or
os.getenv("TESTING_ENVIRONMENT") == "2x",
Expand Down Expand Up @@ -143,3 +145,4 @@ def test_config_appender(self):

# Check field is added
assert output[0]['fields']['foo'] == 'bar'
self.assert_fields_are_documented(output[0])

0 comments on commit 7dbf4cc

Please sign in to comment.