From 5ece9bebcf8baf74d5c318840b4ce747c32c9d37 Mon Sep 17 00:00:00 2001 From: Randy Coburn Date: Wed, 2 Oct 2019 13:47:46 +0200 Subject: [PATCH 1/4] Containers can now be identified by the container ID --- plugins/inputs/docker/docker.go | 16 +++++++++++ plugins/inputs/docker/docker_test.go | 40 ++++++++++++++++++++++++++-- 2 files changed, 54 insertions(+), 2 deletions(-) diff --git a/plugins/inputs/docker/docker.go b/plugins/inputs/docker/docker.go index a3dc78bd49ee9..5983d83d2c15c 100644 --- a/plugins/inputs/docker/docker.go +++ b/plugins/inputs/docker/docker.go @@ -44,6 +44,8 @@ type Docker struct { ContainerStateInclude []string `toml:"container_state_include"` ContainerStateExclude []string `toml:"container_state_exclude"` + ContainerIDAsHostname bool `toml:"container_id_as_hostname"` + Log telegraf.Logger tlsint.ClientConfig @@ -90,6 +92,9 @@ var sampleConfig = ` ## Only collect metrics for these containers, collect all if empty container_names = [] + ## Set the hostname for the metric to the container ID + container_id_as_hostname = false + ## Containers to include and exclude. Globs accepted. ## Note that an empty array for both will include all containers container_name_include = [] @@ -412,6 +417,13 @@ func (d *Docker) gatherInfo(acc telegraf.Accumulator) error { return nil } +func hostnameFromID(id string) string { + if len(id) > 12 { + return id[0:12] + } + return id +} + func (d *Docker) gatherContainer( container types.Container, acc telegraf.Accumulator, @@ -443,6 +455,10 @@ func (d *Docker) gatherContainer( "container_version": imageVersion, } + if d.ContainerIDAsHostname { + tags["host"] = hostnameFromID(container.ID) + } + ctx, cancel := context.WithTimeout(context.Background(), d.Timeout.Duration) defer cancel() diff --git a/plugins/inputs/docker/docker_test.go b/plugins/inputs/docker/docker_test.go index 4add3340d8a5c..2180c5876ff5e 100644 --- a/plugins/inputs/docker/docker_test.go +++ b/plugins/inputs/docker/docker_test.go @@ -629,8 +629,9 @@ func TestContainerStatus(t *testing.T) { return &client, nil } d = Docker{ - Log: testutil.Logger{}, - newClient: newClientFunc, + Log: testutil.Logger{}, + newClient: newClientFunc, + ContainerIDAsHostname: true, } ) @@ -673,6 +674,7 @@ func TestContainerStatus(t *testing.T) { "label2": "test_value_2", "server_version": "17.09.0-ce", "container_status": tt.expect.Status, + "host": "e2173b9478a6", }) }) } @@ -1017,3 +1019,37 @@ func TestContainerName(t *testing.T) { }) } } + +func TestHostnameFromID(t *testing.T) { + tests := []struct { + name string + id string + expect string + }{ + { + name: "Real ID", + id: "565e3a55f5843cfdd4aa5659a1a75e4e78d47f73c3c483f782fe4a26fc8caa07", + expect: "565e3a55f584", + }, + { + name: "Short ID", + id: "shortid123", + expect: "shortid123", + }, + { + name: "No ID", + id: "", + expect: "shortid123", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + output := hostnameFromID(test.id) + if test.expect != output { + t.Logf("Container ID for hostname is wrong. Want: %s, Got: %s", output, test.expect) + } + }) + } + +} From fba5e66ee5115c04a9e22892b3df6c2e20d69a3d Mon Sep 17 00:00:00 2001 From: Randy Coburn Date: Wed, 2 Oct 2019 13:58:17 +0200 Subject: [PATCH 2/4] Updating README and SampleConfig to show changes. --- plugins/inputs/docker/README.md | 14 ++++++++++++++ plugins/inputs/docker/docker.go | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/plugins/inputs/docker/README.md b/plugins/inputs/docker/README.md index 1816107ea974a..15be4c93eb468 100644 --- a/plugins/inputs/docker/README.md +++ b/plugins/inputs/docker/README.md @@ -26,6 +26,9 @@ to gather stats from the [Engine API](https://docs.docker.com/engine/api/v1.24/) ## Deprecated (1.4.0), use container_name_include container_names = [] + ## Set the host tag for the metrics to the container ID hostname, eg first 12 chars + container_id_as_hostname = false + ## Containers to include and exclude. Collect all if empty. Globs accepted. container_name_include = [] container_name_exclude = [] @@ -93,6 +96,17 @@ volumes: - /var/run/docker.sock:/var/run/docker.sock ``` +#### host tag + +Selecting the containers measurements can be tricky if you have many containers with the same name. +To alleviate this issue you can set the below value to true + +```toml +container_id_as_hostname = true +``` + +This will cause all measurements to have the `host` tag be set to the first 12 characters of the container id. The first 12 characters is the common hostname for containers that have no hostname set, as defined by docker. + #### Kubernetes Labels Kubernetes may add many labels to your containers, if they are not needed you diff --git a/plugins/inputs/docker/docker.go b/plugins/inputs/docker/docker.go index 5983d83d2c15c..59167bc6d5237 100644 --- a/plugins/inputs/docker/docker.go +++ b/plugins/inputs/docker/docker.go @@ -92,7 +92,7 @@ var sampleConfig = ` ## Only collect metrics for these containers, collect all if empty container_names = [] - ## Set the hostname for the metric to the container ID + ## Set the host tag for the metrics to the container ID hostname, eg first 12 chars container_id_as_hostname = false ## Containers to include and exclude. Globs accepted. From a5b9128d0c01b844b6ecc0e01df50e7e67c3b9a3 Mon Sep 17 00:00:00 2001 From: Randy Coburn Date: Mon, 7 Oct 2019 11:29:11 +0200 Subject: [PATCH 3/4] Change the tag to be source. Change the readme and configuration samples to be the same. --- plugins/inputs/docker/README.md | 10 +++++----- plugins/inputs/docker/docker.go | 10 +++++----- plugins/inputs/docker/docker_test.go | 8 ++++---- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/plugins/inputs/docker/README.md b/plugins/inputs/docker/README.md index 15be4c93eb468..05a3fff794bcd 100644 --- a/plugins/inputs/docker/README.md +++ b/plugins/inputs/docker/README.md @@ -26,8 +26,8 @@ to gather stats from the [Engine API](https://docs.docker.com/engine/api/v1.24/) ## Deprecated (1.4.0), use container_name_include container_names = [] - ## Set the host tag for the metrics to the container ID hostname, eg first 12 chars - container_id_as_hostname = false + ## Set the source tag for the metrics to the container ID hostname, eg first 12 chars + source_tag = false ## Containers to include and exclude. Collect all if empty. Globs accepted. container_name_include = [] @@ -96,16 +96,16 @@ volumes: - /var/run/docker.sock:/var/run/docker.sock ``` -#### host tag +#### source tag Selecting the containers measurements can be tricky if you have many containers with the same name. To alleviate this issue you can set the below value to true ```toml -container_id_as_hostname = true +source_tag = true ``` -This will cause all measurements to have the `host` tag be set to the first 12 characters of the container id. The first 12 characters is the common hostname for containers that have no hostname set, as defined by docker. +This will cause all measurements to have the `source` tag be set to the first 12 characters of the container id. The first 12 characters is the common hostname for containers that have no explicit hostname set, as defined by docker. #### Kubernetes Labels diff --git a/plugins/inputs/docker/docker.go b/plugins/inputs/docker/docker.go index 59167bc6d5237..02442baf0f2a7 100644 --- a/plugins/inputs/docker/docker.go +++ b/plugins/inputs/docker/docker.go @@ -44,7 +44,7 @@ type Docker struct { ContainerStateInclude []string `toml:"container_state_include"` ContainerStateExclude []string `toml:"container_state_exclude"` - ContainerIDAsHostname bool `toml:"container_id_as_hostname"` + IncludeSourceTag bool `toml:"source_tag"` Log telegraf.Logger @@ -92,8 +92,8 @@ var sampleConfig = ` ## Only collect metrics for these containers, collect all if empty container_names = [] - ## Set the host tag for the metrics to the container ID hostname, eg first 12 chars - container_id_as_hostname = false + ## Set the source tag for the metrics to the container ID hostname, eg first 12 chars + source_tag = false ## Containers to include and exclude. Globs accepted. ## Note that an empty array for both will include all containers @@ -455,8 +455,8 @@ func (d *Docker) gatherContainer( "container_version": imageVersion, } - if d.ContainerIDAsHostname { - tags["host"] = hostnameFromID(container.ID) + if d.IncludeSourceTag { + tags["source"] = hostnameFromID(container.ID) } ctx, cancel := context.WithTimeout(context.Background(), d.Timeout.Duration) diff --git a/plugins/inputs/docker/docker_test.go b/plugins/inputs/docker/docker_test.go index 2180c5876ff5e..148228af47d30 100644 --- a/plugins/inputs/docker/docker_test.go +++ b/plugins/inputs/docker/docker_test.go @@ -629,9 +629,9 @@ func TestContainerStatus(t *testing.T) { return &client, nil } d = Docker{ - Log: testutil.Logger{}, - newClient: newClientFunc, - ContainerIDAsHostname: true, + Log: testutil.Logger{}, + newClient: newClientFunc, + IncludeSourceTag: true, } ) @@ -674,7 +674,7 @@ func TestContainerStatus(t *testing.T) { "label2": "test_value_2", "server_version": "17.09.0-ce", "container_status": tt.expect.Status, - "host": "e2173b9478a6", + "source": "e2173b9478a6", }) }) } From b8176a271e660bb0a557d86717d1c33e040d2b2a Mon Sep 17 00:00:00 2001 From: Randy Coburn Date: Mon, 7 Oct 2019 11:45:41 +0200 Subject: [PATCH 4/4] Adding source tag to docker_logs Updated the README file to reflect the change. --- plugins/inputs/docker/README.md | 2 +- plugins/inputs/docker_log/README.md | 15 +++++++++++++++ plugins/inputs/docker_log/docker_log.go | 15 +++++++++++++++ plugins/inputs/docker_log/docker_log_test.go | 9 ++++++--- 4 files changed, 37 insertions(+), 4 deletions(-) diff --git a/plugins/inputs/docker/README.md b/plugins/inputs/docker/README.md index 05a3fff794bcd..6ec95b64f52c7 100644 --- a/plugins/inputs/docker/README.md +++ b/plugins/inputs/docker/README.md @@ -99,7 +99,7 @@ volumes: #### source tag Selecting the containers measurements can be tricky if you have many containers with the same name. -To alleviate this issue you can set the below value to true +To alleviate this issue you can set the below value to `true` ```toml source_tag = true diff --git a/plugins/inputs/docker_log/README.md b/plugins/inputs/docker_log/README.md index 02f44e14c6960..d2f0dc6144ff9 100644 --- a/plugins/inputs/docker_log/README.md +++ b/plugins/inputs/docker_log/README.md @@ -43,6 +43,9 @@ The docker plugin uses the [Official Docker Client][] to gather logs from the # docker_label_include = [] # docker_label_exclude = [] + ## Set the source tag for the metrics to the container ID hostname, eg first 12 chars + source_tag = false + ## Optional TLS Config # tls_ca = "/etc/telegraf/ca.pem" # tls_cert = "/etc/telegraf/cert.pem" @@ -58,6 +61,17 @@ When using the `"ENV"` endpoint, the connection is configured using the [env]: https://godoc.org/github.com/moby/moby/client#NewEnvClient +### source tag + +Selecting the containers can be tricky if you have many containers with the same name. +To alleviate this issue you can set the below value to `true` + +```toml +source_tag = true +``` + +This will cause all data points to have the `source` tag be set to the first 12 characters of the container id. The first 12 characters is the common hostname for containers that have no explicit hostname set, as defined by docker. + ### Metrics - docker_log @@ -66,6 +80,7 @@ When using the `"ENV"` endpoint, the connection is configured using the - container_version - container_name - stream (stdout, stderr, or tty) + - source - fields: - container_id - message diff --git a/plugins/inputs/docker_log/docker_log.go b/plugins/inputs/docker_log/docker_log.go index 6a675219fc859..81268f5f54e8b 100644 --- a/plugins/inputs/docker_log/docker_log.go +++ b/plugins/inputs/docker_log/docker_log.go @@ -49,6 +49,9 @@ var sampleConfig = ` # docker_label_include = [] # docker_label_exclude = [] + ## Set the source tag for the metrics to the container ID hostname, eg first 12 chars + source_tag = false + ## Optional TLS Config # tls_ca = "/etc/telegraf/ca.pem" # tls_cert = "/etc/telegraf/cert.pem" @@ -82,6 +85,7 @@ type DockerLogs struct { ContainerExclude []string `toml:"container_name_exclude"` ContainerStateInclude []string `toml:"container_state_include"` ContainerStateExclude []string `toml:"container_state_exclude"` + IncludeSourceTag bool `toml:"source_tag"` tlsint.ClientConfig @@ -258,6 +262,10 @@ func (d *DockerLogs) tailContainerLogs( "container_version": imageVersion, } + if d.IncludeSourceTag { + tags["source"] = hostnameFromID(container.ID) + } + // Add matching container labels as tags for k, label := range container.Labels { if d.labelFilter.Match(k) { @@ -435,3 +443,10 @@ func init() { } }) } + +func hostnameFromID(id string) string { + if len(id) > 12 { + return id[0:12] + } + return id +} diff --git a/plugins/inputs/docker_log/docker_log_test.go b/plugins/inputs/docker_log/docker_log_test.go index ce61f6135a9fc..11cf0befd290e 100644 --- a/plugins/inputs/docker_log/docker_log_test.go +++ b/plugins/inputs/docker_log/docker_log_test.go @@ -98,6 +98,7 @@ func Test(t *testing.T) { "container_image": "influxdata/telegraf", "container_version": "1.11.0", "stream": "tty", + "source": "deadbeef", }, map[string]interface{}{ "container_id": "deadbeef", @@ -141,6 +142,7 @@ func Test(t *testing.T) { "container_image": "influxdata/telegraf", "container_version": "1.11.0", "stream": "stdout", + "source": "deadbeef", }, map[string]interface{}{ "container_id": "deadbeef", @@ -155,9 +157,10 @@ func Test(t *testing.T) { t.Run(tt.name, func(t *testing.T) { var acc testutil.Accumulator plugin := &DockerLogs{ - Timeout: internal.Duration{Duration: time.Second * 5}, - newClient: func(string, *tls.Config) (Client, error) { return tt.client, nil }, - containerList: make(map[string]context.CancelFunc), + Timeout: internal.Duration{Duration: time.Second * 5}, + newClient: func(string, *tls.Config) (Client, error) { return tt.client, nil }, + containerList: make(map[string]context.CancelFunc), + IncludeSourceTag: true, } err := plugin.Init()