From 315d1eecc4a7701d1507f173d6f2753134020de7 Mon Sep 17 00:00:00 2001 From: Alex K <8418476+fearful-symmetry@users.noreply.github.com> Date: Mon, 23 Aug 2021 10:18:22 -0700 Subject: [PATCH] fix used memory calculations in docker/memory (#27473) --- .../module/docker/memory/_meta/data.json | 116 +++++++++++------- metricbeat/module/docker/memory/helper.go | 21 +++- .../module/docker/memory/memory_test.go | 28 +++++ 3 files changed, 122 insertions(+), 43 deletions(-) diff --git a/metricbeat/module/docker/memory/_meta/data.json b/metricbeat/module/docker/memory/_meta/data.json index bde42686ea0..7cf34c7736b 100644 --- a/metricbeat/module/docker/memory/_meta/data.json +++ b/metricbeat/module/docker/memory/_meta/data.json @@ -1,61 +1,94 @@ { "@timestamp": "2017-10-12T08:05:34.853Z", "container": { - "id": "aa41902101351f415e6e983b0673c0ba715dd4bc316bd5fc0ebd6fcf94287f86", + "id": "23ce6f1b53181ea3db0611fe4de36f0ebf1c0a37cb8272e028cac06240dafbe0", "image": { - "name": "redis:latest" + "name": "docker.elastic.co/beats/elastic-agent:7.15.0-SNAPSHOT" }, - "name": "amazing_cohen", + "name": "elastic-package-stack_elastic-agent_1", "runtime": "docker" }, "docker": { + "container": { + "labels": { + "com_docker_compose_config-hash": "8e3d03827946685d53a2f171a126c397a3278da18ecd68a970cba9131160c52c", + "com_docker_compose_container-number": "1", + "com_docker_compose_oneoff": "False", + "com_docker_compose_project": "elastic-package-stack", + "com_docker_compose_service": "elastic-agent", + "com_docker_compose_version": "1.28.6", + "description": "Agent manages other beats based on configuration provided.", + "io_k8s_description": "Agent manages other beats based on configuration provided.", + "io_k8s_display-name": "Elastic-Agent image", + "license": "Elastic License", + "maintainer": "infra@elastic.co", + "name": "elastic-agent", + "org_label-schema_build-date": "2021-07-28T09:55:40Z", + "org_label-schema_license": "Elastic License", + "org_label-schema_name": "elastic-agent", + "org_label-schema_schema-version": "1.0", + "org_label-schema_url": "https://www.elastic.co/beats/elastic-agent", + "org_label-schema_vcs-ref": "16108a69f9f437c00cb6125c57bbc01c4eb805bb", + "org_label-schema_vcs-url": "github.com/elastic/beats/v7", + "org_label-schema_vendor": "Elastic", + "org_label-schema_version": "7.15.0-SNAPSHOT", + "org_opencontainers_image_created": "2021-07-28T09:55:40Z", + "org_opencontainers_image_licenses": "Elastic License", + "org_opencontainers_image_title": "Elastic-Agent", + "org_opencontainers_image_vendor": "Elastic", + "release": "1", + "summary": "elastic-agent", + "url": "https://www.elastic.co/beats/elastic-agent", + "vendor": "Elastic", + "version": "7.15.0-SNAPSHOT" + } + }, "memory": { "fail": { "count": 0 }, - "limit": 2095878144, + "limit": 67514433536, "rss": { - "pct": 0.0004025882909345325, - "total": 843776 + "pct": 0, + "total": 0 }, "stats": { - "active_anon": 421888, - "active_file": 36864, - "cache": 86016, - "dirty": 0, - "hierarchical_memory_limit": 9223372036854771712, - "hierarchical_memsw_limit": 9223372036854771712, - "inactive_anon": 421888, - "inactive_file": 49152, - "mapped_file": 53248, - "pgfault": 1587, - "pgmajfault": 1, - "pgpgin": 2426, - "pgpgout": 2199, - "rss": 843776, - "rss_huge": 0, - "total_active_anon": 421888, - "total_active_file": 36864, - "total_cache": 86016, - "total_dirty": 0, - "total_inactive_anon": 421888, - "total_inactive_file": 49152, - "total_mapped_file": 53248, - "total_pgfault": 1587, - "total_pgmajfault": 1, - "total_pgpgin": 2426, - "total_pgpgout": 2199, - "total_rss": 843776, - "total_rss_huge": 0, - "total_unevictable": 0, - "total_writeback": 0, + "active_anon": 270336, + "active_file": 135168, + "anon": 246484992, + "anon_thp": 4194304, + "file": 325484544, + "file_dirty": 0, + "file_mapped": 170582016, + "file_writeback": 0, + "inactive_anon": 250257408, + "inactive_file": 325619712, + "kernel_stack": 2703360, + "pgactivate": 62898, + "pgdeactivate": 0, + "pgfault": 2150971515, + "pglazyfree": 207999, + "pglazyfreed": 0, + "pgmajfault": 0, + "pgrefill": 0, + "pgscan": 0, + "pgsteal": 0, + "shmem": 0, + "slab": 8112800, + "slab_reclaimable": 5753632, + "slab_unreclaimable": 2359168, + "sock": 200704, + "thp_collapse_alloc": 0, + "thp_fault_alloc": 0, "unevictable": 0, - "writeback": 0 + "workingset_activate": 0, + "workingset_nodereclaim": 0, + "workingset_refault": 0 }, "usage": { - "max": 7860224, - "pct": 0.000672283359618831, - "total": 1409024 + "max": 0, + "pct": 0.0039415723433138695, + "total": 266113024 } } }, @@ -65,7 +98,8 @@ "module": "docker" }, "metricset": { - "name": "memory" + "name": "memory", + "period": 10000 }, "service": { "address": "/var/run/docker.sock", diff --git a/metricbeat/module/docker/memory/helper.go b/metricbeat/module/docker/memory/helper.go index 03c93fafed9..ad74726dc92 100644 --- a/metricbeat/module/docker/memory/helper.go +++ b/metricbeat/module/docker/memory/helper.go @@ -62,6 +62,23 @@ func (s *MemoryService) getMemoryStatsList(containers []docker.Stat, dedot bool) func (s *MemoryService) getMemoryStats(myRawStat docker.Stat, dedot bool) MemoryData { totalRSS := myRawStat.Stats.MemoryStats.Stats["total_rss"] + + // Emulate newer docker releases and exclude cache values from memory usage + // See here for a little more context. usage - cache won't work, as it includes shared mappings that can't be dropped + // like regular cache. + // This formula is used by both cadvisor and containerd + // https://github.com/google/cadvisor/commit/307d1b1cb320fef66fab02db749f07a459245451 + var fileUsage, memUsage uint64 + // use this a shortcut to see if the underlying system is V1 or V2 + totalInactive, isV1 := myRawStat.Stats.MemoryStats.Stats["total_inactive_file"] + if isV1 && totalInactive < myRawStat.Stats.MemoryStats.Usage { + fileUsage = totalInactive + } + if fileInactive, ok := myRawStat.Stats.MemoryStats.Stats["inactive_file"]; !isV1 && ok && fileInactive < myRawStat.Stats.MemoryStats.Usage { + fileUsage = fileInactive + } + + memUsage = myRawStat.Stats.MemoryStats.Usage - fileUsage return MemoryData{ Time: common.Time(myRawStat.Stats.Read), Container: docker.NewContainer(myRawStat.Container, dedot), @@ -70,8 +87,8 @@ func (s *MemoryService) getMemoryStats(myRawStat docker.Stat, dedot bool) Memory MaxUsage: myRawStat.Stats.MemoryStats.MaxUsage, TotalRss: totalRSS, TotalRssP: float64(totalRSS) / float64(myRawStat.Stats.MemoryStats.Limit), - Usage: myRawStat.Stats.MemoryStats.Usage, - UsageP: float64(myRawStat.Stats.MemoryStats.Usage) / float64(myRawStat.Stats.MemoryStats.Limit), + Usage: memUsage, + UsageP: float64(memUsage) / float64(myRawStat.Stats.MemoryStats.Limit), Stats: myRawStat.Stats.MemoryStats.Stats, //Windows memory statistics Commit: myRawStat.Stats.MemoryStats.Commit, diff --git a/metricbeat/module/docker/memory/memory_test.go b/metricbeat/module/docker/memory/memory_test.go index 0fd15dc3483..7f1a1eec5eb 100644 --- a/metricbeat/module/docker/memory/memory_test.go +++ b/metricbeat/module/docker/memory/memory_test.go @@ -125,6 +125,34 @@ func TestMemoryServiceBadData(t *testing.T) { } +func TestMemoryMath(t *testing.T) { + memStats := types.StatsJSON{ + Stats: types.Stats{ + Read: time.Now(), + PreCPUStats: types.CPUStats{ + CPUUsage: types.CPUUsage{ + TotalUsage: 200, + }, + }, + MemoryStats: types.MemoryStats{ + Limit: 5, + Usage: 5000, + Stats: map[string]uint64{ + "total_inactive_file": 1000, // CGV1 + "inactive_file": 900, + }, + }, //Test for cases where this is empty + }, + } + + memoryService := &MemoryService{} + memoryRawStats := []docker.Stat{ + docker.Stat{Stats: memStats, Container: &types.Container{Names: []string{"test-container"}, Labels: map[string]string{}}}, + } + rawStats := memoryService.getMemoryStatsList(memoryRawStats, false) + assert.Equal(t, float64(800), rawStats[0].UsageP) // 5000-900 /5 +} + func getMemoryStats(read time.Time, number uint64) types.StatsJSON { myMemoryStats := types.StatsJSON{