diff --git a/events.go b/events.go index 47aa1abe22b..b06587a34e1 100644 --- a/events.go +++ b/events.go @@ -140,6 +140,7 @@ func convertLibcontainerStats(ls *libcontainer.Stats) *types.Stats { s.Memory.Usage = convertMemoryEntry(cg.MemoryStats.Usage) s.Memory.Raw = cg.MemoryStats.Stats s.Memory.PSI = cg.MemoryStats.PSI + s.Memory.MemoryEventCount = cg.MemoryStats.EventCount s.Blkio.IoServiceBytesRecursive = convertBlkioEntry(cg.BlkioStats.IoServiceBytesRecursive) s.Blkio.IoServicedRecursive = convertBlkioEntry(cg.BlkioStats.IoServicedRecursive) diff --git a/libcontainer/cgroups/fs2/fs2.go b/libcontainer/cgroups/fs2/fs2.go index 0760be74b97..d4d95cc0271 100644 --- a/libcontainer/cgroups/fs2/fs2.go +++ b/libcontainer/cgroups/fs2/fs2.go @@ -105,6 +105,10 @@ func (m *Manager) GetStats() (*cgroups.Stats, error) { if err := statMemory(m.dirPath, st); err != nil && !os.IsNotExist(err) { errs = append(errs, err) } + // memory event for CGRoup v2 + if err := eventMemory(m.dirPath, st); err != nil && !os.IsNotExist(err) { + errs = append(errs, err) + } // io (since kernel 4.5) if err := statIo(m.dirPath, st); err != nil && !os.IsNotExist(err) { errs = append(errs, err) diff --git a/libcontainer/cgroups/fs2/memory.go b/libcontainer/cgroups/fs2/memory.go index 29656597423..9077b80b5d2 100644 --- a/libcontainer/cgroups/fs2/memory.go +++ b/libcontainer/cgroups/fs2/memory.go @@ -237,3 +237,19 @@ func rootStatsFromMeminfo(stats *cgroups.Stats) error { return nil } + +func eventMemory(dirPath string, stats *cgroups.Stats) error { + kv, err := fscommon.ParseKeyValueFile(dirPath, "memory.events") + if err != nil { + return err + } + eventCount := cgroups.MemoryEventsCount{} + eventCount.MaxCount = kv["max"] + eventCount.ReclaimLowCount = kv["low"] + eventCount.ReclaimHighCount = kv["high"] + eventCount.OomCount = kv["oom"] + eventCount.OomKillCount = kv["oom_kill"] + stats.MemoryStats.EventCount = eventCount + + return nil +} diff --git a/libcontainer/cgroups/fscommon/utils.go b/libcontainer/cgroups/fscommon/utils.go index f4a51c9e56f..1cad02e6d5b 100644 --- a/libcontainer/cgroups/fscommon/utils.go +++ b/libcontainer/cgroups/fscommon/utils.go @@ -143,3 +143,26 @@ func GetCgroupParamString(path, file string) (string, error) { return strings.TrimSpace(contents), nil } + +// ReadKeyValueFile reads all key-value pairs from the specified cgroup file, +// returns a map from key to value. +func ParseKeyValueFile(dir, file string) (map[string]uint64, error) { + content, err := cgroups.ReadFile(dir, file) + if err != nil { + return nil, err + } + + lines := strings.Split(content, "\n") + vals := make(map[string]uint64, len(lines)) + for _, line := range lines { + arr := strings.Split(line, " ") + if len(arr) == 2 { + val, err := ParseUint(arr[1], 10, 64) + if err == nil { + vals[arr[0]] = val + } + } + } + + return vals, nil +} diff --git a/libcontainer/cgroups/fscommon/utils_test.go b/libcontainer/cgroups/fscommon/utils_test.go index 0339e998904..badb6b1bd16 100644 --- a/libcontainer/cgroups/fscommon/utils_test.go +++ b/libcontainer/cgroups/fscommon/utils_test.go @@ -1,9 +1,11 @@ package fscommon import ( + "errors" "math" "os" "path/filepath" + "reflect" "strconv" "testing" @@ -93,3 +95,84 @@ func TestGetCgroupParamsInt(t *testing.T) { t.Fatal("Expecting error, got none") } } + +func TestParseKeyValueFile(t *testing.T) { + testCases := []struct { + Name string + FileContent []byte + FileExist bool + Filename string + HasErr bool + ExpectedErr error + Expected map[string]uint64 + }{ + { + Name: "Standard memory.events", + FileContent: []byte("low 0\nhigh 0\nmax 12692218\noom 74039\noom_kill 71934\n"), + Filename: "memory.events", + FileExist: true, + HasErr: false, + Expected: map[string]uint64{ + "low": 0, + "high": 0, + "max": 12692218, + "oom": 74039, + "oom_kill": 71934, + }, + }, + { + Name: "File not exists", + FileExist: false, + HasErr: true, + ExpectedErr: os.ErrNotExist, + }, + { + Name: "Sample cpu.stat with invalid line", + FileContent: []byte("usage_usec 27458468773731\nuser_usec 20792829128141\nsystem_usec 6665639645590\n\nval_only\nnon_int xyz\n"), + FileExist: true, + HasErr: false, + Expected: map[string]uint64{ + "usage_usec": 27458468773731, + "user_usec": 20792829128141, + "system_usec": 6665639645590, + }, + }, + } + + for _, testCase := range testCases { + // setup file + tempDir := t.TempDir() + if testCase.Filename == "" { + testCase.Filename = "cgroup.file" + } + + if testCase.FileExist { + tempFile := filepath.Join(tempDir, testCase.Filename) + + if err := os.WriteFile(tempFile, testCase.FileContent, 0o755); err != nil { + t.Fatal(err) + } + } + + // get key value + got, err := ParseKeyValueFile(tempDir, testCase.Filename) + hasErr := err != nil + + // compare expected + if testCase.HasErr != hasErr { + t.Errorf("ParseKeyValueFile returns wrong err: %v for test case: %v", err, testCase.Filename) + } + + if testCase.ExpectedErr != nil && !errors.Is(err, testCase.ExpectedErr) { + t.Errorf("ParseKeyValueFile returns wrong err for test case: %v, expected: %v, got: %v", + testCase.Filename, testCase.Expected, err) + } + + if !testCase.HasErr { + if !reflect.DeepEqual(got, testCase.Expected) { + t.Errorf("ParseKeyValueFile returns wrong result for test case: %v, got: %v, want: %v", + testCase.Filename, got, testCase.Expected) + } + } + } +} diff --git a/libcontainer/cgroups/stats.go b/libcontainer/cgroups/stats.go index b475567d821..b629caf386d 100644 --- a/libcontainer/cgroups/stats.go +++ b/libcontainer/cgroups/stats.go @@ -103,8 +103,22 @@ type MemoryStats struct { // if true, memory usage is accounted for throughout a hierarchy of cgroups. UseHierarchy bool `json:"use_hierarchy"` - Stats map[string]uint64 `json:"stats,omitempty"` - PSI *PSIStats `json:"psi,omitempty"` + Stats map[string]uint64 `json:"stats,omitempty"` + PSI *PSIStats `json:"psi,omitempty"` + EventCount MemoryEventsCount `json:"events_count,omitempty"` +} + +type MemoryEventsCount struct { + // count of memory reclaim (when usage is under the low boundary) + ReclaimLowCount uint64 `json:"reclaim_low_count"` + // count of memory reclaim (when high memory boundary was exceeded) + ReclaimHighCount uint64 `json:"reclaim_high_count"` + // the number of times the cgroup’s memory usage was about to go over the max boundary + MaxCount uint64 `json:"max_count"` + // the number of time the cgroup’s memory usage was reached the limit and allocation was about to fail + OomCount uint64 `json:"oom_count"` + // The number of processes belonging to this cgroup was oom killed + OomKillCount uint64 `json:"oom_kill_count"` } type PageUsageByNUMA struct { diff --git a/tests/integration/events.bats b/tests/integration/events.bats index cf42eaf55cd..4e5fd71477a 100644 --- a/tests/integration/events.bats +++ b/tests/integration/events.bats @@ -121,10 +121,40 @@ function test_events() { retry 10 1 grep -q test_busybox events.log # shellcheck disable=SC2016 __runc exec -d test_busybox sh -c 'test=$(dd if=/dev/urandom ibs=5120k)' - retry 30 1 grep -q oom events.log + retry 30 1 grep -q '{"type":"oom","id":"test_busybox"}' events.log __runc delete -f test_busybox ) & wait # wait for the above sub shells to finish grep -q '{"type":"oom","id":"test_busybox"}' events.log } + +@test "events --stats with OOM memory event" { + requires root cgroups_v2 + init_cgroup_paths + + # we need the container to hit OOM, so disable swap + update_config '(.. | select(.resources? != null)) .resources.memory |= {"limit": 33554432, "swap": 33554432}' + + # run busybox detached + runc run -d --console-socket "$CONSOLE_SOCKET" test_busybox + [ "$status" -eq 0 ] + + # spawn two sub processes (shells) + # the first sub process is an event logger that sends stats events to events.log + # the second sub process exec a memory hog process to cause a oom condition + # and waits for an oom event + (__runc events test_busybox >events.log) & + ( + retry 10 1 grep -q test_busybox events.log + # shellcheck disable=SC2016 + __runc exec -d test_busybox sh -c 'test=$(dd if=/dev/urandom ibs=5120k)' + retry 30 1 grep -q '{"type":"oom","id":"test_busybox"}' events.log + __runc events --stats test_busybox >stats.log + __runc delete -f test_busybox + ) & + wait # wait for the above sub shells to finish + + grep -q '{"type":"oom","id":"test_busybox"}' events.log + jq -e '.data.memory.memory_event_count.oom_kill_count >= 1' <<<"$(cat stats.log)" +} diff --git a/types/events.go b/types/events.go index e28ac8c3836..114d7d0a703 100644 --- a/types/events.go +++ b/types/events.go @@ -28,6 +28,8 @@ type PSIData = cgroups.PSIData type PSIStats = cgroups.PSIStats +type MemoryEventCount = cgroups.MemoryEventsCount + type Hugetlb struct { Usage uint64 `json:"usage,omitempty"` Max uint64 `json:"max,omitempty"` @@ -102,13 +104,14 @@ type MemoryEntry struct { } type Memory struct { - Cache uint64 `json:"cache,omitempty"` - Usage MemoryEntry `json:"usage,omitempty"` - Swap MemoryEntry `json:"swap,omitempty"` - Kernel MemoryEntry `json:"kernel,omitempty"` - KernelTCP MemoryEntry `json:"kernelTCP,omitempty"` - Raw map[string]uint64 `json:"raw,omitempty"` - PSI *PSIStats `json:"psi,omitempty"` + Cache uint64 `json:"cache,omitempty"` + Usage MemoryEntry `json:"usage,omitempty"` + Swap MemoryEntry `json:"swap,omitempty"` + Kernel MemoryEntry `json:"kernel,omitempty"` + KernelTCP MemoryEntry `json:"kernelTCP,omitempty"` + Raw map[string]uint64 `json:"raw,omitempty"` + PSI *PSIStats `json:"psi,omitempty"` + MemoryEventCount MemoryEventCount `json:"memory_event_count"` } type L3CacheInfo struct {