diff --git a/plugins/inputs/docker/docker_test.go b/plugins/inputs/docker/docker_test.go index a84a6047b30aa..42028a572e42c 100644 --- a/plugins/inputs/docker/docker_test.go +++ b/plugins/inputs/docker/docker_test.go @@ -265,6 +265,162 @@ func TestDockerGatherContainerStats(t *testing.T) { acc.AssertDoesNotContainsTaggedFields(t, "docker_container_cpu", cpu3fields, cputags) } +func TestDockerMemoryExcludesCache(t *testing.T) { + var acc testutil.Accumulator + stats := testStats() + + tags := map[string]string{ + "container_name": "redis", + "container_image": "redis/image", + } + + d := &Docker{ + Log: testutil.Logger{}, + } + + delete(stats.MemoryStats.Stats, "cache") + delete(stats.MemoryStats.Stats, "inactive_file") + delete(stats.MemoryStats.Stats, "total_inactive_file") + + // set cgroup v2 cache value + stats.MemoryStats.Stats["inactive_file"] = 9 + + d.parseContainerStats(stats, &acc, tags, "123456789", "linux") + + // test docker_container_mem measurement + memfields := map[string]interface{}{ + "active_anon": uint64(0), + "active_file": uint64(1), + "container_id": "123456789", + "fail_count": uint64(1), + "hierarchical_memory_limit": uint64(0), + "inactive_anon": uint64(0), + "inactive_file": uint64(9), + "limit": uint64(2000), + "mapped_file": uint64(0), + "max_usage": uint64(1001), + "pgfault": uint64(2), + "pgmajfault": uint64(0), + "pgpgin": uint64(0), + "pgpgout": uint64(0), + "rss_huge": uint64(0), + "rss": uint64(0), + "total_active_anon": uint64(0), + "total_active_file": uint64(0), + "total_cache": uint64(0), + "total_inactive_anon": uint64(0), + "total_mapped_file": uint64(0), + "total_pgfault": uint64(0), + "total_pgmajfault": uint64(0), + "total_pgpgin": uint64(4), + "total_pgpgout": uint64(0), + "total_rss_huge": uint64(444), + "total_rss": uint64(44), + "total_unevictable": uint64(0), + "total_writeback": uint64(55), + "unevictable": uint64(0), + "usage_percent": float64(55.1), // 1102 / 2000 + "usage": uint64(1102), + "writeback": uint64(0), + } + + acc.AssertContainsTaggedFields(t, "docker_container_mem", memfields, tags) + acc.ClearMetrics() + + // set cgroup v1 cache value (has priority over cgroups v2) + stats.MemoryStats.Stats["total_inactive_file"] = 7 + + d.parseContainerStats(stats, &acc, tags, "123456789", "linux") + + // test docker_container_mem measurement + memfields = map[string]interface{}{ + "active_anon": uint64(0), + "active_file": uint64(1), + // "cache": uint64(0), + "container_id": "123456789", + "fail_count": uint64(1), + "hierarchical_memory_limit": uint64(0), + "inactive_anon": uint64(0), + "inactive_file": uint64(9), + "limit": uint64(2000), + "mapped_file": uint64(0), + "max_usage": uint64(1001), + "pgfault": uint64(2), + "pgmajfault": uint64(0), + "pgpgin": uint64(0), + "pgpgout": uint64(0), + "rss_huge": uint64(0), + "rss": uint64(0), + "total_active_anon": uint64(0), + "total_active_file": uint64(0), + "total_cache": uint64(0), + "total_inactive_anon": uint64(0), + "total_inactive_file": uint64(7), + "total_mapped_file": uint64(0), + "total_pgfault": uint64(0), + "total_pgmajfault": uint64(0), + "total_pgpgin": uint64(4), + "total_pgpgout": uint64(0), + "total_rss_huge": uint64(444), + "total_rss": uint64(44), + "total_unevictable": uint64(0), + "total_writeback": uint64(55), + "unevictable": uint64(0), + "usage_percent": float64(55.2), // 1104 / 2000 + "usage": uint64(1104), + "writeback": uint64(0), + } + + acc.AssertContainsTaggedFields(t, "docker_container_mem", memfields, tags) + acc.ClearMetrics() + + // set Docker 19.03 and older cache value (has priority over cgroups v1 and v2) + stats.MemoryStats.Stats["cache"] = 16 + + d.parseContainerStats(stats, &acc, tags, "123456789", "linux") + + // test docker_container_mem measurement + memfields = map[string]interface{}{ + "active_anon": uint64(0), + "active_file": uint64(1), + "cache": uint64(16), + "container_id": "123456789", + "fail_count": uint64(1), + "hierarchical_memory_limit": uint64(0), + "inactive_anon": uint64(0), + "inactive_file": uint64(9), + "limit": uint64(2000), + "mapped_file": uint64(0), + "max_usage": uint64(1001), + "pgfault": uint64(2), + "pgmajfault": uint64(0), + "pgpgin": uint64(0), + "pgpgout": uint64(0), + "rss_huge": uint64(0), + "rss": uint64(0), + "total_active_anon": uint64(0), + "total_active_file": uint64(0), + "total_cache": uint64(0), + "total_inactive_anon": uint64(0), + "total_inactive_file": uint64(7), + "total_mapped_file": uint64(0), + "total_pgfault": uint64(0), + "total_pgmajfault": uint64(0), + "total_pgpgin": uint64(4), + "total_pgpgout": uint64(0), + "total_rss_huge": uint64(444), + "total_rss": uint64(44), + "total_unevictable": uint64(0), + "total_writeback": uint64(55), + "unevictable": uint64(0), + "usage_percent": float64(54.75), // 1095 / 2000 + "usage": uint64(1095), + "writeback": uint64(0), + } + + acc.AssertContainsTaggedFields(t, "docker_container_mem", memfields, tags) +} + func TestDocker_WindowsMemoryContainerStats(t *testing.T) { var acc testutil.Accumulator diff --git a/plugins/inputs/docker/stats_helpers.go b/plugins/inputs/docker/stats_helpers.go index 982f131d6d8d3..e5e21ee783154 100644 --- a/plugins/inputs/docker/stats_helpers.go +++ b/plugins/inputs/docker/stats_helpers.go @@ -40,9 +40,28 @@ func calculateCPUPercentWindows(v *types.StatsJSON) float64 { } // CalculateMemUsageUnixNoCache calculate memory usage of the container. -// Page cache is intentionally excluded to avoid misinterpretation of the output. +// Cache is intentionally excluded to avoid misinterpretation of the output. +// +// On Docker 19.03 and older, the result is `mem.Usage - mem.Stats["cache"]`. +// On new docker with cgroup v1 host, the result is `mem.Usage - mem.Stats["total_inactive_file"]`. +// On new docker with cgroup v2 host, the result is `mem.Usage - mem.Stats["inactive_file"]`. +// +// This definition is designed to be consistent with past values and the latest docker CLI +// * https://github.com/docker/cli/blob/6e2838e18645e06f3e4b6c5143898ccc44063e3b/cli/command/container/stats_helpers.go#L239 func CalculateMemUsageUnixNoCache(mem types.MemoryStats) float64 { - return float64(mem.Usage - mem.Stats["cache"]) + // Docker 19.03 and older + if v, isOldDocker := mem.Stats["cache"]; isOldDocker && v < mem.Usage { + return float64(mem.Usage - v) + } + // cgroup v1 + if v, isCgroup1 := mem.Stats["total_inactive_file"]; isCgroup1 && v < mem.Usage { + return float64(mem.Usage - v) + } + // cgroup v2 + if v := mem.Stats["inactive_file"]; v < mem.Usage { + return float64(mem.Usage - v) + } + return float64(mem.Usage) } func CalculateMemPercentUnixNoCache(limit float64, usedNoCache float64) float64 {