From 1a65b78df6d9fbe88cd546d689e3e1af54227f23 Mon Sep 17 00:00:00 2001 From: Dylan Reimerink Date: Tue, 7 Nov 2023 16:09:14 +0100 Subject: [PATCH] btf: Replace `binary.Read` with manual parsing in `readAndInflateTypes` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit During profiling `binary.Read` uses up a significant amount of CPU time. This seems to be due to it using reflection to calculate the amount of bytes to read at runtime and not caching these results. By doing manual `io.ReadFull` calls, pre-calculating struct sizes and reusing types and buffers where possible, we can reduce the CPU time spent in `readAndInflateTypes` by almost 25%. ``` goos: linux goarch: amd64 pkg: github.com/cilium/ebpf/btf cpu: 11th Gen Intel(R) Core(TM) i7-11800H @ 2.30GHz │ before.txt │ after.txt │ │ sec/op │ sec/op vs base │ ParseVmlinux-16 46.08m ± 1% 34.59m ± 2% -24.93% (n=100) │ before.txt │ after.txt │ │ B/op │ B/op vs base │ ParseVmlinux-16 26.65Mi ± 0% 23.49Mi ± 0% -11.87% (n=100) │ before.txt │ after.txt │ │ allocs/op │ allocs/op vs base │ ParseVmlinux-16 467.5k ± 0% 267.7k ± 0% -42.73% (n=100) ``` Signed-off-by: Dylan Reimerink --- btf/btf_types.go | 152 +++++++++++++++++++++++++++++++++++++++++++++++ btf/types.go | 110 +++++++++++++++++++++------------- 2 files changed, 221 insertions(+), 41 deletions(-) diff --git a/btf/btf_types.go b/btf/btf_types.go index f3830061d..f0e327abc 100644 --- a/btf/btf_types.go +++ b/btf/btf_types.go @@ -153,6 +153,19 @@ type btfType struct { SizeType uint32 } +var btfTypeSize = int(unsafe.Sizeof(btfType{})) + +func unmarshalBtfType(bt *btfType, b []byte, bo binary.ByteOrder) (int, error) { + if len(b) < btfTypeSize { + return 0, fmt.Errorf("not enough bytes to unmarshal btfType") + } + + bt.NameOff = bo.Uint32(b[0:]) + bt.Info = bo.Uint32(b[4:]) + bt.SizeType = bo.Uint32(b[8:]) + return btfTypeSize, nil +} + func mask(len uint32) uint32 { return (1 << len) - 1 } @@ -300,6 +313,17 @@ const ( btfIntBitsShift = 0 ) +var btfIntLen = int(unsafe.Sizeof(btfInt{})) + +func unmarshalBtfInt(bi *btfInt, b []byte, bo binary.ByteOrder) (int, error) { + if len(b) < btfIntLen { + return 0, fmt.Errorf("not enough bytes to unmarshal btfInt") + } + + bi.Raw = bo.Uint32(b[0:]) + return btfIntLen, nil +} + func (bi btfInt) Encoding() IntEncoding { return IntEncoding(readBits(bi.Raw, btfIntEncodingLen, btfIntEncodingShift)) } @@ -330,38 +354,166 @@ type btfArray struct { Nelems uint32 } +var btfArrayLen = int(unsafe.Sizeof(btfArray{})) + +func unmarshalBtfArray(ba *btfArray, b []byte, bo binary.ByteOrder) (int, error) { + if len(b) < btfArrayLen { + return 0, fmt.Errorf("not enough bytes to unmarshal btfArray") + } + + ba.Type = TypeID(bo.Uint32(b[0:])) + ba.IndexType = TypeID(bo.Uint32(b[4:])) + ba.Nelems = bo.Uint32(b[8:]) + return btfArrayLen, nil +} + type btfMember struct { NameOff uint32 Type TypeID Offset uint32 } +var btfMemberLen = int(unsafe.Sizeof(btfMember{})) + +func unmarshalBtfMembers(members []btfMember, b []byte, bo binary.ByteOrder) (int, error) { + off := 0 + for i := range members { + if off+btfMemberLen > len(b) { + return 0, fmt.Errorf("not enough bytes to unmarshal btfMember %d", i) + } + + members[i].NameOff = bo.Uint32(b[off+0:]) + members[i].Type = TypeID(bo.Uint32(b[off+4:])) + members[i].Offset = bo.Uint32(b[off+8:]) + + off += btfMemberLen + } + + return off, nil +} + type btfVarSecinfo struct { Type TypeID Offset uint32 Size uint32 } +var btfVarSecinfoLen = int(unsafe.Sizeof(btfVarSecinfo{})) + +func unmarshalBtfVarSecInfos(secinfos []btfVarSecinfo, b []byte, bo binary.ByteOrder) (int, error) { + off := 0 + for i := range secinfos { + if off+btfVarSecinfoLen > len(b) { + return 0, fmt.Errorf("not enough bytes to unmarshal btfVarSecinfo %d", i) + } + + secinfos[i].Type = TypeID(bo.Uint32(b[off+0:])) + secinfos[i].Offset = bo.Uint32(b[off+4:]) + secinfos[i].Size = bo.Uint32(b[off+8:]) + + off += btfVarSecinfoLen + } + + return off, nil +} + type btfVariable struct { Linkage uint32 } +var btfVariableLen = int(unsafe.Sizeof(btfVariable{})) + +func unmarshalBtfVariable(bv *btfVariable, b []byte, bo binary.ByteOrder) (int, error) { + if len(b) < btfVariableLen { + return 0, fmt.Errorf("not enough bytes to unmarshal btfVariable") + } + + bv.Linkage = bo.Uint32(b[0:]) + return btfVariableLen, nil +} + type btfEnum struct { NameOff uint32 Val uint32 } +var btfEnumLen = int(unsafe.Sizeof(btfEnum{})) + +func unmarshalBtfEnums(enums []btfEnum, b []byte, bo binary.ByteOrder) (int, error) { + off := 0 + for i := range enums { + if off+btfEnumLen > len(b) { + return 0, fmt.Errorf("not enough bytes to unmarshal btfEnum %d", i) + } + + enums[i].NameOff = bo.Uint32(b[off+0:]) + enums[i].Val = bo.Uint32(b[off+4:]) + + off += btfEnumLen + } + + return off, nil +} + type btfEnum64 struct { NameOff uint32 ValLo32 uint32 ValHi32 uint32 } +var btfEnum64Len = int(unsafe.Sizeof(btfEnum64{})) + +func unmarshalBtfEnums64(enums []btfEnum64, b []byte, bo binary.ByteOrder) (int, error) { + off := 0 + for i := range enums { + if off+btfEnum64Len > len(b) { + return 0, fmt.Errorf("not enough bytes to unmarshal btfEnum64 %d", i) + } + + enums[i].NameOff = bo.Uint32(b[off+0:]) + enums[i].ValLo32 = bo.Uint32(b[off+4:]) + enums[i].ValHi32 = bo.Uint32(b[off+8:]) + + off += btfEnum64Len + } + + return off, nil +} + type btfParam struct { NameOff uint32 Type TypeID } +var btfParamLen = int(unsafe.Sizeof(btfParam{})) + +func unmarshalBtfParams(params []btfParam, b []byte, bo binary.ByteOrder) (int, error) { + off := 0 + for i := range params { + if off+btfParamLen > len(b) { + return 0, fmt.Errorf("not enough bytes to unmarshal btfParam %d", i) + } + + params[i].NameOff = bo.Uint32(b[off+0:]) + params[i].Type = TypeID(bo.Uint32(b[off+4:])) + + off += btfParamLen + } + + return off, nil +} + type btfDeclTag struct { ComponentIdx uint32 } + +var btfDeclTagLen = int(unsafe.Sizeof(btfDeclTag{})) + +func unmarshalBtfDeclTag(bdt *btfDeclTag, b []byte, bo binary.ByteOrder) (int, error) { + if len(b) < btfDeclTagLen { + return 0, fmt.Errorf("not enough bytes to unmarshal btfDeclTag") + } + + bdt.ComponentIdx = bo.Uint32(b[0:]) + return btfDeclTagLen, nil +} diff --git a/btf/types.go b/btf/types.go index 5be7cc0e2..8bd7fc77f 100644 --- a/btf/types.go +++ b/btf/types.go @@ -11,6 +11,7 @@ import ( "github.com/cilium/ebpf/asm" "github.com/cilium/ebpf/internal" "github.com/cilium/ebpf/internal/sys" + "golang.org/x/exp/slices" ) // Mirrors MAX_RESOLVE_DEPTH in libbpf. @@ -848,6 +849,7 @@ func readAndInflateTypes(r io.Reader, bo binary.ByteOrder, typeLen uint32, rawSt } var ( + buf = make([]byte, 1024) header btfType bInt btfInt bArr btfArray @@ -867,12 +869,16 @@ func readAndInflateTypes(r io.Reader, bo binary.ByteOrder, typeLen uint32, rawSt typ Type ) - if err := binary.Read(r, bo, &header); err == io.EOF { + if _, err := io.ReadFull(r, buf[:btfTypeLen]); err == io.EOF { break } else if err != nil { return nil, fmt.Errorf("can't read type info for id %v: %v", id, err) } + if _, err := unmarshalBtfType(&header, buf[:btfTypeLen], bo); err != nil { + return nil, fmt.Errorf("can't unmarshal type info for id %v: %v", id, err) + } + if id < firstTypeID { return nil, fmt.Errorf("no more type IDs") } @@ -885,9 +891,13 @@ func readAndInflateTypes(r io.Reader, bo binary.ByteOrder, typeLen uint32, rawSt switch header.Kind() { case kindInt: size := header.Size() - if err := binary.Read(r, bo, &bInt); err != nil { + buf = buf[:btfIntLen] + if _, err := io.ReadFull(r, buf); err != nil { return nil, fmt.Errorf("can't read btfInt, id: %d: %w", id, err) } + if _, err := unmarshalBtfInt(&bInt, buf, bo); err != nil { + return nil, fmt.Errorf("can't unmarshal btfInt, id: %d: %w", id, err) + } if bInt.Offset() > 0 || bInt.Bits().Bytes() != size { legacyBitfields[id] = [2]Bits{bInt.Offset(), bInt.Bits()} } @@ -899,9 +909,13 @@ func readAndInflateTypes(r io.Reader, bo binary.ByteOrder, typeLen uint32, rawSt typ = ptr case kindArray: - if err := binary.Read(r, bo, &bArr); err != nil { + buf = buf[:btfArrayLen] + if _, err := io.ReadFull(r, buf); err != nil { return nil, fmt.Errorf("can't read btfArray, id: %d: %w", id, err) } + if _, err := unmarshalBtfArray(&bArr, buf, bo); err != nil { + return nil, fmt.Errorf("can't unmarshal btfArray, id: %d: %w", id, err) + } arr := &Array{nil, nil, bArr.Nelems} fixup(bArr.IndexType, &arr.Index) @@ -910,15 +924,16 @@ func readAndInflateTypes(r io.Reader, bo binary.ByteOrder, typeLen uint32, rawSt case kindStruct: vlen := header.Vlen() - if len(bMembers) < vlen { - bMembers = append(bMembers, make([]btfMember, vlen-len(bMembers))...) - } - - if err := binary.Read(r, bo, bMembers[:vlen]); err != nil { + bMembers = slices.Grow(bMembers[:0], vlen)[:vlen] + buf = slices.Grow(buf[:0], vlen*btfMemberLen)[:vlen*btfMemberLen] + if _, err := io.ReadFull(r, buf); err != nil { return nil, fmt.Errorf("can't read btfMembers, id: %d: %w", id, err) } + if _, err := unmarshalBtfMembers(bMembers, buf, bo); err != nil { + return nil, fmt.Errorf("can't unmarshal btfMembers, id: %d: %w", id, err) + } - members, err := convertMembers(bMembers[:vlen], header.Bitfield()) + members, err := convertMembers(bMembers, header.Bitfield()) if err != nil { return nil, fmt.Errorf("struct %s (id %d): %w", name, id, err) } @@ -926,15 +941,16 @@ func readAndInflateTypes(r io.Reader, bo binary.ByteOrder, typeLen uint32, rawSt case kindUnion: vlen := header.Vlen() - if len(bMembers) < vlen { - bMembers = append(bMembers, make([]btfMember, vlen-len(bMembers))...) - } - - if err := binary.Read(r, bo, bMembers[:vlen]); err != nil { + bMembers = slices.Grow(bMembers[:0], vlen)[:vlen] + buf = slices.Grow(buf[:0], vlen*btfMemberLen)[:vlen*btfMemberLen] + if _, err := io.ReadFull(r, buf); err != nil { return nil, fmt.Errorf("can't read btfMembers, id: %d: %w", id, err) } + if _, err := unmarshalBtfMembers(bMembers, buf, bo); err != nil { + return nil, fmt.Errorf("can't unmarshal btfMembers, id: %d: %w", id, err) + } - members, err := convertMembers(bMembers[:vlen], header.Bitfield()) + members, err := convertMembers(bMembers, header.Bitfield()) if err != nil { return nil, fmt.Errorf("union %s (id %d): %w", name, id, err) } @@ -942,17 +958,18 @@ func readAndInflateTypes(r io.Reader, bo binary.ByteOrder, typeLen uint32, rawSt case kindEnum: vlen := header.Vlen() - if len(bEnums) < vlen { - bEnums = append(bEnums, make([]btfEnum, vlen-len(bEnums))...) + bEnums = slices.Grow(bEnums[:0], vlen)[:vlen] + buf = slices.Grow(buf[:0], vlen*btfEnumLen)[:vlen*btfEnumLen] + if _, err := io.ReadFull(r, buf); err != nil { + return nil, fmt.Errorf("can't read btfEnums, id: %d: %w", id, err) } - - if err := binary.Read(r, bo, bEnums[:vlen]); err != nil { - return nil, fmt.Errorf("can't read btfEnum, id: %d: %w", id, err) + if _, err := unmarshalBtfEnums(bEnums, buf, bo); err != nil { + return nil, fmt.Errorf("can't unmarshal btfEnums, id: %d: %w", id, err) } vals := make([]EnumValue, 0, vlen) signed := header.Signed() - for i, btfVal := range bEnums[:vlen] { + for i, btfVal := range bEnums { name, err := rawStrings.Lookup(btfVal.NameOff) if err != nil { return nil, fmt.Errorf("get name for enum value %d: %s", i, err) @@ -996,16 +1013,17 @@ func readAndInflateTypes(r io.Reader, bo binary.ByteOrder, typeLen uint32, rawSt case kindFuncProto: vlen := header.Vlen() - if len(bParams) < vlen { - bParams = append(bParams, make([]btfParam, vlen-len(bParams))...) + bParams = slices.Grow(bParams[:0], vlen)[:vlen] + buf = slices.Grow(buf[:0], vlen*btfParamLen)[:vlen*btfParamLen] + if _, err := io.ReadFull(r, buf); err != nil { + return nil, fmt.Errorf("can't read btfParams, id: %d: %w", id, err) } - - if err := binary.Read(r, bo, bParams[:vlen]); err != nil { - return nil, fmt.Errorf("can't read btfParam, id: %d: %w", id, err) + if _, err := unmarshalBtfParams(bParams, buf, bo); err != nil { + return nil, fmt.Errorf("can't unmarshal btfParams, id: %d: %w", id, err) } params := make([]FuncParam, 0, vlen) - for i, param := range bParams[:vlen] { + for i, param := range bParams { name, err := rawStrings.Lookup(param.NameOff) if err != nil { return nil, fmt.Errorf("get name for func proto parameter %d: %s", i, err) @@ -1023,7 +1041,11 @@ func readAndInflateTypes(r io.Reader, bo binary.ByteOrder, typeLen uint32, rawSt typ = fp case kindVar: - if err := binary.Read(r, bo, &bVariable); err != nil { + buf = buf[:btfVariableLen] + if _, err := io.ReadFull(r, buf); err != nil { + return nil, fmt.Errorf("can't read btfVariable, id: %d: %w", id, err) + } + if _, err := unmarshalBtfVariable(&bVariable, buf, bo); err != nil { return nil, fmt.Errorf("can't read btfVariable, id: %d: %w", id, err) } @@ -1033,16 +1055,17 @@ func readAndInflateTypes(r io.Reader, bo binary.ByteOrder, typeLen uint32, rawSt case kindDatasec: vlen := header.Vlen() - if len(bSecInfos) < vlen { - bSecInfos = append(bSecInfos, make([]btfVarSecinfo, vlen-len(bSecInfos))...) + bSecInfos = slices.Grow(bSecInfos[:0], vlen)[:vlen] + buf = slices.Grow(buf[:0], vlen*btfVarSecinfoLen)[:vlen*btfVarSecinfoLen] + if _, err := io.ReadFull(r, buf); err != nil { + return nil, fmt.Errorf("can't read btfVarSecInfos, id: %d: %w", id, err) } - - if err := binary.Read(r, bo, bSecInfos[:vlen]); err != nil { - return nil, fmt.Errorf("can't read btfVarSecinfo, id: %d: %w", id, err) + if _, err := unmarshalBtfVarSecInfos(bSecInfos, buf, bo); err != nil { + return nil, fmt.Errorf("can't unmarshal btfVarSecInfos, id: %d: %w", id, err) } vars := make([]VarSecinfo, 0, vlen) - for _, btfVar := range bSecInfos[:vlen] { + for _, btfVar := range bSecInfos { vars = append(vars, VarSecinfo{ Offset: btfVar.Offset, Size: btfVar.Size, @@ -1057,7 +1080,11 @@ func readAndInflateTypes(r io.Reader, bo binary.ByteOrder, typeLen uint32, rawSt typ = &Float{name, header.Size()} case kindDeclTag: - if err := binary.Read(r, bo, &bDeclTag); err != nil { + buf = buf[:btfDeclTagLen] + if _, err := io.ReadFull(r, buf); err != nil { + return nil, fmt.Errorf("can't read btfDeclTag, id: %d: %w", id, err) + } + if _, err := unmarshalBtfDeclTag(&bDeclTag, buf, bo); err != nil { return nil, fmt.Errorf("can't read btfDeclTag, id: %d: %w", id, err) } @@ -1079,16 +1106,17 @@ func readAndInflateTypes(r io.Reader, bo binary.ByteOrder, typeLen uint32, rawSt case kindEnum64: vlen := header.Vlen() - if len(bEnums64) < vlen { - bEnums64 = append(bEnums64, make([]btfEnum64, vlen-len(bEnums64))...) + bEnums64 = slices.Grow(bEnums64[:0], vlen)[:vlen] + buf = slices.Grow(buf[:0], vlen*btfEnum64Len)[:vlen*btfEnum64Len] + if _, err := io.ReadFull(r, buf); err != nil { + return nil, fmt.Errorf("can't read btfEnum64s, id: %d: %w", id, err) } - - if err := binary.Read(r, bo, bEnums64[:vlen]); err != nil { - return nil, fmt.Errorf("can't read btfEnum64, id: %d: %w", id, err) + if _, err := unmarshalBtfEnums64(bEnums64, buf, bo); err != nil { + return nil, fmt.Errorf("can't unmarshal btfEnum64s, id: %d: %w", id, err) } vals := make([]EnumValue, 0, vlen) - for i, btfVal := range bEnums64[:vlen] { + for i, btfVal := range bEnums64 { name, err := rawStrings.Lookup(btfVal.NameOff) if err != nil { return nil, fmt.Errorf("get name for enum64 value %d: %s", i, err)