From 52504e9295f81e2dcaf00665d522162f0e4819d0 Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Mon, 30 Sep 2024 11:55:41 -0400 Subject: [PATCH 01/10] [cdac] Implement NibbleMap lookup and tests The execution manager uses a nibble map to quickly map program counter pointers to the beginnings of the native code for the managed method. Implement the lookup algorithm for a nibble map. Start adding unit tests for the nibble map Also for testing in MockMemorySpace simplify ReaderContext, there's nothing special about the descriptor HeapFragments anymore. We can use a uniform reader. --- docs/design/datacontracts/ExecutionManager.md | 21 ++ .../DataType.cs | 1 + .../Target.cs | 7 + .../TargetCodePointer.cs | 30 +++ .../TargetNUInt.cs | 5 + .../TargetPointer.cs | 2 + .../ExecutionManagerHelpers/NibbleMap.cs | 179 +++++++++++++ .../ContractDescriptorTarget.cs | 14 + .../cdacreader/tests/MockMemorySpace.cs | 50 +--- .../cdacreader/tests/NibbleMapTests.cs | 192 ++++++++++++++ .../cdacreader/tests/TestPlaceholderTarget.cs | 240 ++++++++++++++++++ 11 files changed, 701 insertions(+), 40 deletions(-) create mode 100644 docs/design/datacontracts/ExecutionManager.md create mode 100644 src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Abstractions/TargetCodePointer.cs create mode 100644 src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/ExecutionManagerHelpers/NibbleMap.cs create mode 100644 src/native/managed/cdacreader/tests/NibbleMapTests.cs create mode 100644 src/native/managed/cdacreader/tests/TestPlaceholderTarget.cs diff --git a/docs/design/datacontracts/ExecutionManager.md b/docs/design/datacontracts/ExecutionManager.md new file mode 100644 index 00000000000000..ce59d3ac8c8bba --- /dev/null +++ b/docs/design/datacontracts/ExecutionManager.md @@ -0,0 +1,21 @@ +# Contract ExecutionManager + +This contract is for mapping a PC adddress to information about the +managed method corresponding to that address. + + +## APIs of contract + +**TODO** + +## Version 1 + +**TODO** Methods + +### NibbleMap + +Version 1 of this contract depends on a "nibble map" data structure +that allows mapping of a code address in a contiguous subsection of +the address space to the pointer to the start of that a code sequence. +It takes advantage of the fact that the code starts are aligned and +are spaced apart to represent their addresses as a 4-bit nibble value. diff --git a/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Abstractions/DataType.cs b/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Abstractions/DataType.cs index 3d9926eb4f505d..36f1c5312bf50a 100644 --- a/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Abstractions/DataType.cs +++ b/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Abstractions/DataType.cs @@ -20,6 +20,7 @@ public enum DataType pointer, GCHandle, + CodePointer, Thread, ThreadStore, GCAllocContext, diff --git a/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Abstractions/Target.cs b/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Abstractions/Target.cs index 46128d9dc59e0d..26ff88bdacc09a 100644 --- a/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Abstractions/Target.cs +++ b/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Abstractions/Target.cs @@ -41,6 +41,13 @@ internal abstract class Target /// Pointer read from the target} public abstract TargetPointer ReadPointer(ulong address); + /// + /// Read a code pointer from the target in target endianness + /// + /// Address to start reading from + /// Pointer read from the target} + public abstract TargetCodePointer ReadCodePointer(ulong address); + /// /// Read some bytes from the target /// diff --git a/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Abstractions/TargetCodePointer.cs b/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Abstractions/TargetCodePointer.cs new file mode 100644 index 00000000000000..42de55def6f7ee --- /dev/null +++ b/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Abstractions/TargetCodePointer.cs @@ -0,0 +1,30 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +using System; + +namespace Microsoft.Diagnostics.DataContractReader; + +public readonly struct TargetCodePointer : IEquatable +{ + public static TargetCodePointer Null = new(0); + public readonly ulong Value; + public TargetCodePointer(ulong value) => Value = value; + + public static implicit operator ulong(TargetCodePointer p) => p.Value; + public static implicit operator TargetCodePointer(ulong v) => new TargetCodePointer(v); + + public static bool operator ==(TargetCodePointer left, TargetCodePointer right) => left.Value == right.Value; + public static bool operator !=(TargetCodePointer left, TargetCodePointer right) => left.Value != right.Value; + + public override bool Equals(object? obj) => obj is TargetCodePointer pointer && Equals(pointer); + public bool Equals(TargetCodePointer other) => Value == other.Value; + + public override int GetHashCode() => Value.GetHashCode(); + + public bool Equals(TargetCodePointer x, TargetCodePointer y) => x.Value == y.Value; + public int GetHashCode(TargetCodePointer obj) => obj.Value.GetHashCode(); + + public TargetPointer AsTargetPointer => new(Value); + + public override string ToString() => $"0x{Value:x}"; +} diff --git a/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Abstractions/TargetNUInt.cs b/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Abstractions/TargetNUInt.cs index bbacb9a50b94fd..8a565225ba8311 100644 --- a/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Abstractions/TargetNUInt.cs +++ b/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Abstractions/TargetNUInt.cs @@ -1,11 +1,16 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. using System; +using System.Diagnostics; namespace Microsoft.Diagnostics.DataContractReader; + +[DebuggerDisplay("{Hex}")] public readonly struct TargetNUInt { public readonly ulong Value; public TargetNUInt(ulong value) => Value = value; + + internal string Hex => $"0x{Value:x}"; } diff --git a/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Abstractions/TargetPointer.cs b/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Abstractions/TargetPointer.cs index 9aef236554a38b..d145347f3220a9 100644 --- a/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Abstractions/TargetPointer.cs +++ b/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Abstractions/TargetPointer.cs @@ -23,4 +23,6 @@ namespace Microsoft.Diagnostics.DataContractReader; public bool Equals(TargetPointer other) => Value == other.Value; public override int GetHashCode() => Value.GetHashCode(); + + public override string ToString() => $"0x{Value:x}"; } diff --git a/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/ExecutionManagerHelpers/NibbleMap.cs b/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/ExecutionManagerHelpers/NibbleMap.cs new file mode 100644 index 00000000000000..2c68f62079962a --- /dev/null +++ b/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/ExecutionManagerHelpers/NibbleMap.cs @@ -0,0 +1,179 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Numerics; + +using MapUnit = uint; + + +namespace Microsoft.Diagnostics.DataContractReader.ExecutionManagerHelpers; + +// Given a contiguous region of memory in which we lay out a collection of non-overlapping code blocks that are +// not too small (so that two adjacent ones aren't too close together) and where the start of each code block is preceeded by a code header aligned on some power of 2, +// we can break up the whole memory space into buckets of a fixed size (32-bytes in the current implementation), where +// each bucket either has a code block header or not. +// Thinking of each code block header address as a hex number, we can view it as: [index, offset, zeros] +// where each index gives us a bucket and the offset gives us the position of the header within the bucket. +// We encode each offset into a 4-bit nibble, reserving the special value 0 to mark the places in the map where a method doesn't start. +// +// To find the start of a method given an address we first convert it into a bucket index (giving the map unit) +// and an offset which we can then turn into the index of the nibble that covers that address. +// If the nibble is non-zero, we have the start of a method and it is near the given address. +// If the nibble is zero, we have to search backward first through the current map unit, and then through previous map +// units until we find a non-zero nibble. +#pragma warning disable SA1121 // Use built in alias +internal sealed class NibbleMap +{ + + public static NibbleMap Create(Target target) + { + uint codeHeaderSize = (uint)target.PointerSize; + return new NibbleMap(target, codeHeaderSize); + } + + private readonly Target _target; + private readonly uint _codeHeaderSize; + private NibbleMap(Target target, uint codeHeaderSize) + { + _target = target; + _codeHeaderSize = codeHeaderSize; + } + + // Shift the next nibble into the least significant position. + private static T NextNibble(T n) where T : IBinaryInteger + { + return n >>> 4; + } + + + private const uint MapUnitBytes = sizeof(MapUnit); // our units are going to be 32-bit integers + private const MapUnit NibbleMask = 0x0F; + private const ulong NibblesPerMapUnit = 2 * MapUnitBytes; // 2 nibbles per byte * N bytes per map unit + + // we will partition the address space into buckets of this many bytes. + // There is at most one code block header per bucket. + // normally we would then need Log2(BytesPerBucket) bits to find the exact start address, + // but because code headers are aligned, we can store the offset in a 4-bit nibble instead and shift appropriately to compute + // the effective address + private const ulong BytesPerBucket = 8 * sizeof(MapUnit); + + + // given the index of a nibble in the map, compute how much we have to shift a MapUnit to put that + // nible in the least significant position. + private static int ComputeNibbleShift(ulong mapIdx) + { + // the low bits of the nibble index give us how many nibbles we have to shift by. + int nibbleOffsetInMapUnit = (int)(mapIdx & (NibblesPerMapUnit - 1)); + return 28 - (nibbleOffsetInMapUnit * 4); // bit shift - 4 bits per nibble + } + + private static ulong ComputeByteOffset(ulong mapIdx, uint nibble) + { + return mapIdx * BytesPerBucket + (nibble - 1) * MapUnitBytes; + } + private static TargetPointer GetAbsoluteAddress(TargetPointer baseAddress, ulong mapIdx, uint nibble) + { + return baseAddress + ComputeByteOffset(mapIdx, nibble); + } + + // Given a relative address, decompose it into + // the bucket index and an offset within the bucket. + private static void DecomposeAddress(TargetNUInt relative, out ulong mapIdx, out uint bucketByteIndex) + { + mapIdx = relative.Value / BytesPerBucket; + bucketByteIndex = ((uint)(relative.Value & (BytesPerBucket - 1)) / MapUnitBytes) + 1; + System.Diagnostics.Debug.Assert(bucketByteIndex == (bucketByteIndex & NibbleMask)); + } + + private static TargetPointer GetMapUnitAddress(TargetPointer mapStart, ulong mapIdx) + { + return mapStart + (mapIdx / NibblesPerMapUnit) * MapUnitBytes; + } + + internal static TargetPointer RoundTripAddress(TargetPointer mapBase, TargetPointer currentPC) + { + TargetNUInt relativeAddress = new TargetNUInt(currentPC.Value - mapBase.Value); + DecomposeAddress(relativeAddress, out ulong mapIdx, out uint bucketByteIndex); + return mapBase + ComputeByteOffset(mapIdx, bucketByteIndex); + } + + internal TargetPointer FindMethodCode(TargetPointer mapBase, TargetPointer mapStart, TargetCodePointer currentPC) + { + TargetNUInt relativeAddress = new TargetNUInt(currentPC.Value - mapBase.Value); + DecomposeAddress(relativeAddress, out ulong mapIdx, out uint bucketByteIndex); + + MapUnit t = _target.Read(GetMapUnitAddress(mapStart, mapIdx)); + + // shift the nibble we want to the least significant position + t = t >>> ComputeNibbleShift(mapIdx); + uint nibble = t & NibbleMask; + if (nibble != 0 && nibble <= bucketByteIndex) + { + return GetAbsoluteAddress(mapBase, mapIdx, nibble); + } + + // search backwards through the current map unit + // we processed the lsb nibble, move to the next one + t = NextNibble(t); + + // if there's any nibble set in the current unit, find it + if (t != 0) + { + mapIdx--; + nibble = t & NibbleMask; + while (nibble == 0) + { + t = NextNibble(t); + mapIdx--; + nibble = t & NibbleMask; + } + return GetAbsoluteAddress(mapBase, mapIdx, nibble); + } + + // if we were near the beginning of the address space, there is not enough space for the code header, + // so we can stop + if (mapIdx < NibblesPerMapUnit) + { + return TargetPointer.Null; + } + + // We're now done with the current map index. + // Align the map index and move to the previous map unit, then move back one nibble. +#pragma warning disable IDE0054 // use compound assignment + mapIdx = mapIdx & (~(NibblesPerMapUnit - 1)); + mapIdx--; +#pragma warning restore IDE0054 // use compound assignment + + // read the map unit containing mapIdx and skip over it if it is all zeros + while (mapIdx >= NibblesPerMapUnit) + { + t = _target.Read(GetMapUnitAddress(mapStart, mapIdx)); + if (t != 0) + break; + mapIdx -= NibblesPerMapUnit; + } + + // if we went all the way to the front, we didn't find a code header + if (mapIdx < NibblesPerMapUnit) + { + return TargetPointer.Null; + } + + // move to the correct nibble in the map unit + while (mapIdx != 0 && (t & NibbleMask) == 0) + { + t = NextNibble(t); + mapIdx--; + } + + if (mapIdx == 0 && t == 0) + { + return TargetPointer.Null; + } + + nibble = t & NibbleMask; + return GetAbsoluteAddress(mapBase, mapIdx, nibble); + } + +} +#pragma warning restore SA1121 // Use built in alias diff --git a/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader/ContractDescriptorTarget.cs b/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader/ContractDescriptorTarget.cs index 3c3b4c67118393..965c0a1507aec5 100644 --- a/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader/ContractDescriptorTarget.cs +++ b/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader/ContractDescriptorTarget.cs @@ -298,6 +298,20 @@ public override TargetPointer ReadPointerFromSpan(ReadOnlySpan bytes) } } + public override TargetCodePointer ReadCodePointer(ulong address) + { + TypeInfo codePointerTypeInfo = GetTypeInfo(DataType.CodePointer); + if (codePointerTypeInfo.Size is sizeof(uint)) + { + return new TargetCodePointer(Read(address)); + } + else if (codePointerTypeInfo.Size is sizeof(ulong)) + { + return new TargetCodePointer(Read(address)); + } + throw new InvalidOperationException($"Failed to read code pointer at 0x{address:x8} because CodePointer size is not 4 or 8"); + } + public void ReadPointers(ulong address, Span buffer) { // TODO(cdac) - This could do a single read, and then swizzle in place if it is useful for performance diff --git a/src/native/managed/cdacreader/tests/MockMemorySpace.cs b/src/native/managed/cdacreader/tests/MockMemorySpace.cs index 645d251dbbea99..084a6dbe06c8b1 100644 --- a/src/native/managed/cdacreader/tests/MockMemorySpace.cs +++ b/src/native/managed/cdacreader/tests/MockMemorySpace.cs @@ -188,7 +188,7 @@ private string MakeContractsJson() return (json, pointerData); } - private ReadContext CreateContext() + private ReadContext CreateContext(out ulong contractDescriptorAddress) { if (_created) throw new InvalidOperationException("Context already created"); @@ -198,14 +198,17 @@ private ReadContext CreateContext() HeapFragment descriptor = CreateContractDescriptor(json.Data.Length, pointerDataCount); + AddHeapFragment(descriptor); + AddHeapFragment(json); + if (pointerData.Data.Length > 0) + AddHeapFragment(pointerData); + ReadContext context = new ReadContext { - ContractDescriptor = descriptor, - JsonDescriptor = json, - PointerData = pointerData, HeapFragments = _heapFragments, }; _created = true; + contractDescriptorAddress = descriptor.Address; return context; } @@ -228,53 +231,20 @@ private bool FragmentFits(HeapFragment f) public bool TryCreateTarget([NotNullWhen(true)] out ContractDescriptorTarget? target) { - ReadContext context = CreateContext(); - return ContractDescriptorTarget.TryCreate(context.ContractDescriptor.Address, context.ReadFromTarget, out target); + ReadContext context = CreateContext(out ulong contractDescriptorAddress); + return ContractDescriptorTarget.TryCreate(contractDescriptorAddress, context.ReadFromTarget, out target); } } // Used by ReadFromTarget to return the appropriate bytes internal class ReadContext { - public HeapFragment ContractDescriptor { get; init;} - - public HeapFragment JsonDescriptor { get; init; } - - public HeapFragment PointerData { get; init;} public IReadOnlyList HeapFragments { get; init; } - internal int ReadFromTarget(ulong address, Span span) + internal int ReadFromTarget(ulong address, Span buffer) { if (address == 0) return -1; - // Populate the span with the requested portion of the contract descriptor - if (address >= ContractDescriptor.Address && address + (uint)span.Length <= ContractDescriptor.Address + (ulong)ContractDescriptor.Data.Length) - { - int offset = checked ((int)(address - ContractDescriptor.Address)); - ContractDescriptor.Data.AsSpan(offset, span.Length).CopyTo(span); - return 0; - } - - // Populate the span with the JSON descriptor - this assumes the product will read it all at once. - if (address == JsonDescriptor.Address) - { - JsonDescriptor.Data.AsSpan().CopyTo(span); - return 0; - } - - // Populate the span with the requested portion of the pointer data - if (address >= PointerData.Address && address + (uint)span.Length <= PointerData.Address + (ulong)PointerData.Data.Length) - { - int offset = checked((int)(address - PointerData.Address)); - PointerData.Data.AsSpan(offset, span.Length).CopyTo(span); - return 0; - } - - return ReadFragment(address, span); - } - - private int ReadFragment(ulong address, Span buffer) - { bool partialReadOcurred = false; HeapFragment lastHeapFragment = default; int availableLength = 0; diff --git a/src/native/managed/cdacreader/tests/NibbleMapTests.cs b/src/native/managed/cdacreader/tests/NibbleMapTests.cs new file mode 100644 index 00000000000000..f75f3048cd97df --- /dev/null +++ b/src/native/managed/cdacreader/tests/NibbleMapTests.cs @@ -0,0 +1,192 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using Xunit; + +using Microsoft.Diagnostics.DataContractReader.ExecutionManagerHelpers; +using System.Diagnostics; + +namespace Microsoft.Diagnostics.DataContractReader.UnitTests; + +public class NibbleMapTests +{ + internal class NibbleMapTestTarget : TestPlaceholderTarget + { + private readonly MockMemorySpace.ReadContext _readContext; + public NibbleMapTestTarget(MockTarget.Architecture arch, MockMemorySpace.ReadContext readContext) : base(arch) + { + _readContext = readContext; + SetDataReader(_readContext.ReadFromTarget); + } + + } + + internal class NibbleMapTestBuilder + { + // This is the base address of the memory range that the map covers. + // The map works on code pointers as offsets from this address + // For testing we don't actually place anything into this space + private readonly TargetPointer MapBase; + + + private readonly MockTarget.Architecture Arch; + // this is the target memory representation of the nibble map itself + public readonly MockMemorySpace.HeapFragment NibbleMapFragment; + + public NibbleMapTestBuilder(TargetPointer mapBase, ulong mapRangeSize, TargetPointer mapStart,MockTarget.Architecture arch) + { + MapBase = mapBase; + Arch = arch; + int nibbleMapSize = (int)Addr2Pos(mapRangeSize); + NibbleMapFragment = new MockMemorySpace.HeapFragment { + Address = mapStart, + Data = new byte[nibbleMapSize], + Name = "Nibble Map", + }; + } + + const int Log2CodeAlign = 2; // N.B. this might be different on 64-bit in the future + const int Log2NibblesPerDword = 3; + const int Log2BytesPerBucket = Log2CodeAlign + Log2NibblesPerDword; + const int Log2NibbleSize = 2; + const int NibbleSize = 1 << Log2NibbleSize; + const uint NibblesPerDword = (8 * sizeof(uint)) >> Log2NibbleSize; + const uint NibblesPerDwordMask = NibblesPerDword - 1; + const uint BytesPerBucket = NibblesPerDword * (1 << Log2CodeAlign); + + const uint MaskBytesPerBucket = BytesPerBucket - 1; + + const uint NibbleMask = 0xf; + const int HighestNibbleBit = 32 - NibbleSize; + + const uint HighestNibbleMask = NibbleMask << HighestNibbleBit; + + private ulong Addr2Pos(ulong addr) + { + return addr >> Log2BytesPerBucket; + } + + private uint Addr2Offs(ulong addr) + { + return (uint) (((addr & MaskBytesPerBucket) >> Log2CodeAlign) + 1); + } + + private int Pos2ShiftCount (ulong addr) + { + return HighestNibbleBit - (int)((addr & NibblesPerDwordMask) << Log2NibbleSize); + } + public void AllocateCodeChunk(TargetCodePointer codeStart, int codeSize) + { + // paraphrased from EEJitManager::NibbleMapSetUnlocked + if (codeStart.Value < MapBase.Value) + { + throw new ArgumentException("Code start address is below the map base"); + } + ulong delta = codeStart.Value - MapBase.Value; + ulong pos = Addr2Pos(delta); + bool bSet = true; + uint value = bSet?Addr2Offs(delta):0; + + uint index = (uint) (pos >> Log2NibblesPerDword); + uint mask = ~(HighestNibbleMask >> (int)((pos & NibblesPerDwordMask) << Log2NibbleSize)); + + value = value << Pos2ShiftCount(pos); + + Span entry = NibbleMapFragment.Data.AsSpan((int)(index * sizeof(uint)), sizeof(uint)); + uint oldValue = TestPlaceholderTarget.ReadFromSpan(entry, Arch.IsLittleEndian); + + if (value != 0 && (oldValue & ~mask) != 0) + { + throw new InvalidOperationException("Overwriting existing offset"); + } + + uint newValue = (oldValue & mask) | value; + TestPlaceholderTarget.WriteToSpan(newValue, Arch.IsLittleEndian, entry); + } + + public NibbleMapTestTarget Create() + { + return new NibbleMapTestTarget(Arch, new MockMemorySpace.ReadContext() { + HeapFragments = new[] { NibbleMapFragment } + }); + } + } + + [Fact] + public void RoundTripAddressTest() + { + TargetPointer mapBase = 0; + uint delta = 0x10u; + for (TargetPointer p = mapBase; p < mapBase + 0x1000; p += delta) + { + TargetPointer actual = NibbleMap.RoundTripAddress(mapBase, p); + Assert.Equal(p, actual); + } + } + + [Theory] + [ClassData(typeof(MockTarget.StdArch))] + public void NibbleMapOneItemLookupOk(MockTarget.Architecture arch) + { + // SETUP: + + // this is the beginning of the address range where code pointers might point + TargetPointer mapBase = new(0x5f5f_0000u); + // this is the beginning of the nibble map itself + TargetPointer mapStart = new(0x0456_1000u); + /// this is how big the address space is that the map covers + const uint MapRangeSize = 0x1000; + TargetPointer MapEnd = mapBase + MapRangeSize; + NibbleMapTestBuilder builder = new(mapBase, MapRangeSize, mapStart, arch); + + // don't put the code too close to the start - the NibbleMap bails if the code is too close to the start of the range + TargetCodePointer inputPC = new(mapBase + 0x0200u); + int codeSize = 0x80; // doesn't matter + builder.AllocateCodeChunk (inputPC, codeSize); + NibbleMapTestTarget target = builder.Create(); + + // TODO: some kind of memory in the placeholder target + //target.AddHeapFragment(builder.NibbleMapFragment); + + // TESTCASE: + + NibbleMap map = NibbleMap.Create(target); + Assert.NotNull(map); + + TargetPointer methodCode = map.FindMethodCode(mapBase, mapStart, inputPC); + Assert.Equal(inputPC.Value, methodCode.Value); + + // All addresses in the code chunk should map to the same method + for (int i = 0; i < codeSize; i++) + { + methodCode = map.FindMethodCode(mapBase, mapStart, inputPC.Value + (uint)i); + // we should always find the beginning of the method + Assert.Equal(inputPC.Value, methodCode.Value); + } + + // All addresses before the code chunk should return null + for (ulong i = mapBase; i < inputPC; i++) + { + methodCode = map.FindMethodCode(mapBase, mapStart, i); + Assert.Equal(0u, methodCode.Value); + } + + methodCode = map.FindMethodCode(mapBase, mapStart, inputPC.Value + 0x100u); + Assert.Equal(inputPC.Value, methodCode.Value); + +#if true // FIXME: for expected 5f5f0200, found 5f5f0120 input address 5f5f0300 + + // interestingly, all addresses after the code chunk should also return the beginning of the method + // we don't track how long the method is, so we can't tell if we're past the end + for (TargetCodePointer ptr = inputPC + (uint)codeSize; ptr < MapEnd; ptr++) + { + methodCode = map.FindMethodCode(mapBase, mapStart, ptr); + Assert.Equal(inputPC.Value, methodCode); + } +#endif + + } + + +} diff --git a/src/native/managed/cdacreader/tests/TestPlaceholderTarget.cs b/src/native/managed/cdacreader/tests/TestPlaceholderTarget.cs new file mode 100644 index 00000000000000..6d496f979214fa --- /dev/null +++ b/src/native/managed/cdacreader/tests/TestPlaceholderTarget.cs @@ -0,0 +1,240 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Numerics; +using System.Runtime.CompilerServices; + +namespace Microsoft.Diagnostics.DataContractReader.UnitTests; + +/// +/// A base class implementation of Target that throws NotImplementedException for all methods. +/// +internal class TestPlaceholderTarget : Target +{ + private protected ContractRegistry contractRegistry; + private protected Target.IDataCache dataCache; + private protected Dictionary typeInfoCache; + + internal delegate int ReadFromTargetDelegate(ulong address, Span buffer); + + protected ReadFromTargetDelegate _dataReader = (address, buffer) => throw new NotImplementedException(); + +#region Setup + public TestPlaceholderTarget(MockTarget.Architecture arch) + { + IsLittleEndian = arch.IsLittleEndian; + PointerSize = arch.Is64Bit ? 8 : 4; + contractRegistry = new TestRegistry();; + dataCache = new TestDataCache(); + typeInfoCache = null; + } + + internal void SetContracts(ContractRegistry contracts) + { + contractRegistry = contracts; + } + + internal void SetDataCache(Target.IDataCache cache) + { + dataCache = cache; + } + + internal void SetTypeInfoCache(Dictionary cache) + { + typeInfoCache = cache; + } + + internal void SetDataReader(ReadFromTargetDelegate reader) + { + _dataReader = reader; + } +#endregion Setup + + public override int PointerSize { get; } + public override bool IsLittleEndian { get; } + + public override bool IsAlignedToPointerSize(TargetPointer pointer) + { + return (pointer.Value & (ulong)(PointerSize - 1)) == 0; + } + + public override TargetPointer ReadGlobalPointer(string global) => throw new NotImplementedException(); + public override TargetPointer ReadPointer(ulong address) => DefaultReadPointer(address); + public override TargetCodePointer ReadCodePointer(ulong address) => throw new NotImplementedException(); + public override void ReadBuffer(ulong address, Span buffer) => throw new NotImplementedException(); + public override string ReadUtf8String(ulong address) => throw new NotImplementedException(); + public override string ReadUtf16String(ulong address) => throw new NotImplementedException(); + public override TargetNUInt ReadNUInt(ulong address) => DefaultReadNUInt(address); + public override T ReadGlobal(string name) => throw new NotImplementedException(); + public override T Read(ulong address) => DefaultRead(address); + +#region subclass reader helpers + + /// + /// Basic utility to read a value from memory, all the DefaultReadXXX methods call this. + /// + protected unsafe bool DefaultTryRead(ulong address, out T value) where T : unmanaged, IBinaryInteger, IMinMaxValue + { + value = default; + Span buffer = stackalloc byte[sizeof(T)]; + if (_dataReader(address, buffer) < 0) + return false; + + value = ReadFromSpan(buffer, IsLittleEndian); + return true; + } + + internal unsafe static T ReadFromSpan(ReadOnlySpan bytes, bool isLittleEndian) where T : unmanaged, IBinaryInteger, IMinMaxValue + { + if (sizeof(T) != bytes.Length) + throw new ArgumentException(nameof(bytes)); + + T value; + if (isLittleEndian) + { + T.TryReadLittleEndian(bytes, !IsSigned(), out value); + } + else + { + T.TryReadBigEndian(bytes, !IsSigned(), out value); + } + return value; + } + + internal unsafe static void WriteToSpan(T value, bool isLittleEndian, Span dest) where T : unmanaged, IBinaryInteger, IMinMaxValue + { + if (sizeof(T) != dest.Length) + throw new ArgumentException(nameof(dest)); + + if (isLittleEndian) + { + value.WriteLittleEndian(dest); + } + else + { + value.WriteBigEndian(dest); + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + protected static bool IsSigned() where T : struct, INumberBase, IMinMaxValue + { + return T.IsNegative(T.MinValue); + } + + protected T DefaultRead(ulong address) where T : unmanaged, IBinaryInteger, IMinMaxValue + { + if (!DefaultTryRead(address, out T value)) + throw new InvalidOperationException($"Failed to read {typeof(T)} at 0x{address:x8}."); + return value; + } + + protected TargetPointer DefaultReadPointer (ulong address) + { + if (!DefaultTryReadPointer(address, out TargetPointer pointer)) + throw new InvalidOperationException($"Failed to read pointer at 0x{address:x8}."); + + return pointer; + } + + protected bool DefaultTryReadPointer(ulong address, out TargetPointer pointer) + { + pointer = TargetPointer.Null; + if (!DefaultTryReadNUInt(address, out ulong value)) + return false; + + pointer = new TargetPointer(value); + return true; + } + + protected bool DefaultTryReadNUInt(ulong address, out ulong value) + { + value = 0; + if (PointerSize == sizeof(uint) + && DefaultTryRead(address, out uint value32)) + { + value = value32; + return true; + } + else if (PointerSize == sizeof(ulong) + && DefaultTryRead(address, out ulong value64)) + { + value = value64; + return true; + } + + return false; + } + + protected TargetNUInt DefaultReadNUInt(ulong address) + { + if (!DefaultTryReadNUInt(address, out ulong value)) + throw new InvalidOperationException($"Failed to read nuint at 0x{address:x8}."); + + return new TargetNUInt(value); + } +#endregion subclass reader helpers + + public override TargetPointer ReadPointerFromSpan(ReadOnlySpan bytes) => throw new NotImplementedException(); + + public override Target.TypeInfo GetTypeInfo(DataType dataType) => typeInfoCache != null ? GetTypeInfoImpl(dataType) : throw new NotImplementedException(); + + private protected virtual Target.TypeInfo GetTypeInfoImpl(DataType dataType) + { + if (typeInfoCache!.TryGetValue(dataType, out var info)) + { + return info; + } + throw new NotImplementedException(); + } + + public override Target.IDataCache ProcessedData => dataCache; + public override ContractRegistry Contracts => contractRegistry; + + internal class TestRegistry : ContractRegistry + { + public TestRegistry() { } + internal Contracts.IException? ExceptionContract { get; set; } + internal Contracts.ILoader? LoaderContract { get; set; } + internal Contracts.IEcmaMetadata? EcmaMetadataContract { get; set; } + internal Contracts.IObject? ObjectContract { get; set; } + internal Contracts.IThread? ThreadContract { get; set; } + internal Contracts.IRuntimeTypeSystem? RuntimeTypeSystemContract { get; set; } + internal Contracts.IDacStreams? DacStreamsContract { get; set; } + + public override Contracts.IException Exception => ExceptionContract ?? throw new NotImplementedException(); + public override Contracts.ILoader Loader => LoaderContract ?? throw new NotImplementedException(); + public override Contracts.IEcmaMetadata EcmaMetadata => EcmaMetadataContract ?? throw new NotImplementedException(); + public override Contracts.IObject Object => ObjectContract ?? throw new NotImplementedException(); + public override Contracts.IThread Thread => ThreadContract ?? throw new NotImplementedException(); + public override Contracts.IRuntimeTypeSystem RuntimeTypeSystem => RuntimeTypeSystemContract ?? throw new NotImplementedException(); + public override Contracts.IDacStreams DacStreams => DacStreamsContract ?? throw new NotImplementedException(); + } + + internal class TestDataCache : Target.IDataCache + { + public TestDataCache() {} + + public virtual T GetOrAdd(TargetPointer address) where T : Data.IData + { + if (TryGet(address.Value, out T? data)) + { + return data; + } + return Add(address.Value); + } + + public virtual bool TryGet(ulong address, [NotNullWhen(true)] out T? data) + { + throw new NotImplementedException(); + } + + protected virtual T Add(ulong address) where T : Data.IData + { + throw new NotImplementedException(); + } + } +} From 43fb677cbeb3cecf8309b4d08ce5677874784733 Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Mon, 30 Sep 2024 13:31:02 -0400 Subject: [PATCH 02/10] remove fixed fixme --- src/native/managed/cdacreader/tests/NibbleMapTests.cs | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/native/managed/cdacreader/tests/NibbleMapTests.cs b/src/native/managed/cdacreader/tests/NibbleMapTests.cs index f75f3048cd97df..fa8a0d31e90c22 100644 --- a/src/native/managed/cdacreader/tests/NibbleMapTests.cs +++ b/src/native/managed/cdacreader/tests/NibbleMapTests.cs @@ -175,8 +175,6 @@ public void NibbleMapOneItemLookupOk(MockTarget.Architecture arch) methodCode = map.FindMethodCode(mapBase, mapStart, inputPC.Value + 0x100u); Assert.Equal(inputPC.Value, methodCode.Value); -#if true // FIXME: for expected 5f5f0200, found 5f5f0120 input address 5f5f0300 - // interestingly, all addresses after the code chunk should also return the beginning of the method // we don't track how long the method is, so we can't tell if we're past the end for (TargetCodePointer ptr = inputPC + (uint)codeSize; ptr < MapEnd; ptr++) @@ -184,7 +182,6 @@ public void NibbleMapOneItemLookupOk(MockTarget.Architecture arch) methodCode = map.FindMethodCode(mapBase, mapStart, ptr); Assert.Equal(inputPC.Value, methodCode); } -#endif } From dc3c1a43530a4d5b7a2fa29a8d772a2ce35c4a28 Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Mon, 30 Sep 2024 13:32:55 -0400 Subject: [PATCH 03/10] remove one more TODO --- src/native/managed/cdacreader/tests/NibbleMapTests.cs | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/native/managed/cdacreader/tests/NibbleMapTests.cs b/src/native/managed/cdacreader/tests/NibbleMapTests.cs index fa8a0d31e90c22..174622e3d602fb 100644 --- a/src/native/managed/cdacreader/tests/NibbleMapTests.cs +++ b/src/native/managed/cdacreader/tests/NibbleMapTests.cs @@ -146,9 +146,6 @@ public void NibbleMapOneItemLookupOk(MockTarget.Architecture arch) builder.AllocateCodeChunk (inputPC, codeSize); NibbleMapTestTarget target = builder.Create(); - // TODO: some kind of memory in the placeholder target - //target.AddHeapFragment(builder.NibbleMapFragment); - // TESTCASE: NibbleMap map = NibbleMap.Create(target); From 26204402738d8dd671022e50fb72c6d37d7069a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleksey=20Kliger=20=28=CE=BBgeek=29?= Date: Tue, 1 Oct 2024 09:55:51 -0400 Subject: [PATCH 04/10] spelling Co-authored-by: Filip Navara --- docs/design/datacontracts/ExecutionManager.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/design/datacontracts/ExecutionManager.md b/docs/design/datacontracts/ExecutionManager.md index ce59d3ac8c8bba..eb986235463b3d 100644 --- a/docs/design/datacontracts/ExecutionManager.md +++ b/docs/design/datacontracts/ExecutionManager.md @@ -1,6 +1,6 @@ # Contract ExecutionManager -This contract is for mapping a PC adddress to information about the +This contract is for mapping a PC address to information about the managed method corresponding to that address. From 2a822997070a90f5cfd88baf2adc10be9e7610da Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Fri, 4 Oct 2024 16:08:32 -0400 Subject: [PATCH 05/10] NibbleMap: remove unused _codeHeaderSize field --- .../ExecutionManagerHelpers/NibbleMap.cs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/ExecutionManagerHelpers/NibbleMap.cs b/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/ExecutionManagerHelpers/NibbleMap.cs index 2c68f62079962a..4d4562e7c4eb06 100644 --- a/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/ExecutionManagerHelpers/NibbleMap.cs +++ b/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/ExecutionManagerHelpers/NibbleMap.cs @@ -27,16 +27,13 @@ internal sealed class NibbleMap public static NibbleMap Create(Target target) { - uint codeHeaderSize = (uint)target.PointerSize; - return new NibbleMap(target, codeHeaderSize); + return new NibbleMap(target); } private readonly Target _target; - private readonly uint _codeHeaderSize; - private NibbleMap(Target target, uint codeHeaderSize) + private NibbleMap(Target target) { _target = target; - _codeHeaderSize = codeHeaderSize; } // Shift the next nibble into the least significant position. From 10a2b6a20edfe068f69db412f4e5198cf42a59ed Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Fri, 4 Oct 2024 16:41:02 -0400 Subject: [PATCH 06/10] cleanup NibbleMap and tests --- .../ExecutionManagerHelpers/NibbleMap.cs | 19 +++++++------- .../cdacreader/tests/NibbleMapTests.cs | 26 +++++++++++++++++++ 2 files changed, 36 insertions(+), 9 deletions(-) diff --git a/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/ExecutionManagerHelpers/NibbleMap.cs b/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/ExecutionManagerHelpers/NibbleMap.cs index 4d4562e7c4eb06..af767e1433011b 100644 --- a/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/ExecutionManagerHelpers/NibbleMap.cs +++ b/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/ExecutionManagerHelpers/NibbleMap.cs @@ -37,11 +37,7 @@ private NibbleMap(Target target) } // Shift the next nibble into the least significant position. - private static T NextNibble(T n) where T : IBinaryInteger - { - return n >>> 4; - } - + private static MapUnit NextNibble (MapUnit n) => n >>> 4; private const uint MapUnitBytes = sizeof(MapUnit); // our units are going to be 32-bit integers private const MapUnit NibbleMask = 0x0F; @@ -49,25 +45,29 @@ private static T NextNibble(T n) where T : IBinaryInteger // we will partition the address space into buckets of this many bytes. // There is at most one code block header per bucket. - // normally we would then need Log2(BytesPerBucket) bits to find the exact start address, + // Normally we would then need 5 bits (Log2(BytesPerBucket))to find the exact start address, // but because code headers are aligned, we can store the offset in a 4-bit nibble instead and shift appropriately to compute // the effective address private const ulong BytesPerBucket = 8 * sizeof(MapUnit); // given the index of a nibble in the map, compute how much we have to shift a MapUnit to put that - // nible in the least significant position. - private static int ComputeNibbleShift(ulong mapIdx) + // nibble in the least significant position. + internal static int ComputeNibbleShift(ulong mapIdx) { // the low bits of the nibble index give us how many nibbles we have to shift by. int nibbleOffsetInMapUnit = (int)(mapIdx & (NibblesPerMapUnit - 1)); return 28 - (nibbleOffsetInMapUnit * 4); // bit shift - 4 bits per nibble } + // Given a map index and a nibble index, compute the byte offset in the address space that they represent private static ulong ComputeByteOffset(ulong mapIdx, uint nibble) { return mapIdx * BytesPerBucket + (nibble - 1) * MapUnitBytes; } + + // Given a base address, a map index, and a nibble index, compute the absolute address in memory + // that the index and nibble point to. private static TargetPointer GetAbsoluteAddress(TargetPointer baseAddress, ulong mapIdx, uint nibble) { return baseAddress + ComputeByteOffset(mapIdx, nibble); @@ -82,6 +82,7 @@ private static void DecomposeAddress(TargetNUInt relative, out ulong mapIdx, out System.Diagnostics.Debug.Assert(bucketByteIndex == (bucketByteIndex & NibbleMask)); } + // Given a logical index into the map, compute the address in memory where that map unit is located private static TargetPointer GetMapUnitAddress(TargetPointer mapStart, ulong mapIdx) { return mapStart + (mapIdx / NibblesPerMapUnit) * MapUnitBytes; @@ -91,7 +92,7 @@ internal static TargetPointer RoundTripAddress(TargetPointer mapBase, TargetPoin { TargetNUInt relativeAddress = new TargetNUInt(currentPC.Value - mapBase.Value); DecomposeAddress(relativeAddress, out ulong mapIdx, out uint bucketByteIndex); - return mapBase + ComputeByteOffset(mapIdx, bucketByteIndex); + return GetAbsoluteAddress(mapBase, mapIdx, bucketByteIndex); } internal TargetPointer FindMethodCode(TargetPointer mapBase, TargetPointer mapStart, TargetCodePointer currentPC) diff --git a/src/native/managed/cdacreader/tests/NibbleMapTests.cs b/src/native/managed/cdacreader/tests/NibbleMapTests.cs index 174622e3d602fb..666202bc4c492b 100644 --- a/src/native/managed/cdacreader/tests/NibbleMapTests.cs +++ b/src/native/managed/cdacreader/tests/NibbleMapTests.cs @@ -125,6 +125,32 @@ public void RoundTripAddressTest() } } + [Theory] + [InlineData(0u)] + [InlineData(0x100u)] + [InlineData(0xab00u)] + // we don't really expct nibble maps to be this huge... + [InlineData(0xabcd_abcd_7fff_ff00u)] + public void ExhaustiveNibbbleShifts(ulong irrelevant) + { + // Try all possible inputs to ComputeNibbleShift. + // Given the index of a nibble in the map, compute how much we have to shift a MapUnit to put that + // nibble in the least significant position. + // Actually we could just go up to 31 (since a map unit is 32 bits), but we'll go up to 255 for good measure + int expectedShift = 28; + for (int i = 0; i < 255; i++) + { + ulong input = irrelevant + (ulong)i; + int actualShift = NibbleMap.ComputeNibbleShift(input); + Assert.True(expectedShift == actualShift, $"Expected {expectedShift}, got {actualShift} for input 0x{input:x}"); + expectedShift -= 4; + if (expectedShift == -4) + { + expectedShift = 28; + } + } + } + [Theory] [ClassData(typeof(MockTarget.StdArch))] public void NibbleMapOneItemLookupOk(MockTarget.Architecture arch) From e71aa09efaedceef3227cedf0caded8921e4ed26 Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Mon, 7 Oct 2024 09:58:11 -0400 Subject: [PATCH 07/10] NibbleMap: use a struct for MapUnit --- .../ExecutionManagerHelpers/NibbleMap.cs | 87 +++++++++++-------- 1 file changed, 52 insertions(+), 35 deletions(-) diff --git a/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/ExecutionManagerHelpers/NibbleMap.cs b/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/ExecutionManagerHelpers/NibbleMap.cs index af767e1433011b..4de19ab27f5c7d 100644 --- a/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/ExecutionManagerHelpers/NibbleMap.cs +++ b/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/ExecutionManagerHelpers/NibbleMap.cs @@ -3,9 +3,6 @@ using System.Numerics; -using MapUnit = uint; - - namespace Microsoft.Diagnostics.DataContractReader.ExecutionManagerHelpers; // Given a contiguous region of memory in which we lay out a collection of non-overlapping code blocks that are @@ -21,9 +18,29 @@ namespace Microsoft.Diagnostics.DataContractReader.ExecutionManagerHelpers; // If the nibble is non-zero, we have the start of a method and it is near the given address. // If the nibble is zero, we have to search backward first through the current map unit, and then through previous map // units until we find a non-zero nibble. -#pragma warning disable SA1121 // Use built in alias internal sealed class NibbleMap { + // The contents of the map are 32-bit integers, where each nibble is a 4-bit integer. + internal readonly struct MapUnit + { + public const int SizeInBytes = sizeof(uint); + public const ulong SizeInNibbles = 2 * SizeInBytes; + public readonly uint Value; + + public MapUnit(uint value) => Value = value; + + public override string ToString() => Value.ToString(); + + public MapUnit LogicalShiftRight(int bits) => new MapUnit(Value >> bits); + + // Shift the next nibble into the least significant position. + public MapUnit ShiftNextNibble => new MapUnit(Value >>> 4); + + public const uint NibbleMask = 0x0Fu; + public uint Nibble => Value & NibbleMask; + + public bool IsZero => Value == 0; + } public static NibbleMap Create(Target target) { @@ -36,19 +53,14 @@ private NibbleMap(Target target) _target = target; } - // Shift the next nibble into the least significant position. - private static MapUnit NextNibble (MapUnit n) => n >>> 4; - private const uint MapUnitBytes = sizeof(MapUnit); // our units are going to be 32-bit integers - private const MapUnit NibbleMask = 0x0F; - private const ulong NibblesPerMapUnit = 2 * MapUnitBytes; // 2 nibbles per byte * N bytes per map unit // we will partition the address space into buckets of this many bytes. // There is at most one code block header per bucket. // Normally we would then need 5 bits (Log2(BytesPerBucket))to find the exact start address, // but because code headers are aligned, we can store the offset in a 4-bit nibble instead and shift appropriately to compute // the effective address - private const ulong BytesPerBucket = 8 * sizeof(MapUnit); + private const ulong BytesPerBucket = 8 * MapUnit.SizeInBytes; // given the index of a nibble in the map, compute how much we have to shift a MapUnit to put that @@ -56,14 +68,14 @@ private NibbleMap(Target target) internal static int ComputeNibbleShift(ulong mapIdx) { // the low bits of the nibble index give us how many nibbles we have to shift by. - int nibbleOffsetInMapUnit = (int)(mapIdx & (NibblesPerMapUnit - 1)); + int nibbleOffsetInMapUnit = (int)(mapIdx & (MapUnit.SizeInNibbles - 1)); return 28 - (nibbleOffsetInMapUnit * 4); // bit shift - 4 bits per nibble } // Given a map index and a nibble index, compute the byte offset in the address space that they represent private static ulong ComputeByteOffset(ulong mapIdx, uint nibble) { - return mapIdx * BytesPerBucket + (nibble - 1) * MapUnitBytes; + return mapIdx * BytesPerBucket + (nibble - 1) * MapUnit.SizeInBytes; } // Given a base address, a map index, and a nibble index, compute the absolute address in memory @@ -78,14 +90,15 @@ private static TargetPointer GetAbsoluteAddress(TargetPointer baseAddress, ulong private static void DecomposeAddress(TargetNUInt relative, out ulong mapIdx, out uint bucketByteIndex) { mapIdx = relative.Value / BytesPerBucket; - bucketByteIndex = ((uint)(relative.Value & (BytesPerBucket - 1)) / MapUnitBytes) + 1; - System.Diagnostics.Debug.Assert(bucketByteIndex == (bucketByteIndex & NibbleMask)); + bucketByteIndex = ((uint)(relative.Value & (BytesPerBucket - 1)) / MapUnit.SizeInBytes) + 1; + System.Diagnostics.Debug.Assert(bucketByteIndex == (bucketByteIndex & MapUnit.NibbleMask)); } // Given a logical index into the map, compute the address in memory where that map unit is located private static TargetPointer GetMapUnitAddress(TargetPointer mapStart, ulong mapIdx) { - return mapStart + (mapIdx / NibblesPerMapUnit) * MapUnitBytes; + ulong bucketIndex = mapIdx / MapUnit.SizeInNibbles; + return mapStart + bucketIndex * MapUnit.SizeInBytes; } internal static TargetPointer RoundTripAddress(TargetPointer mapBase, TargetPointer currentPC) @@ -95,16 +108,21 @@ internal static TargetPointer RoundTripAddress(TargetPointer mapBase, TargetPoin return GetAbsoluteAddress(mapBase, mapIdx, bucketByteIndex); } + private MapUnit ReadMapUnit(TargetPointer mapStart, ulong mapIdx) + { + return new MapUnit (_target.Read(GetMapUnitAddress(mapStart, mapIdx))); + } + internal TargetPointer FindMethodCode(TargetPointer mapBase, TargetPointer mapStart, TargetCodePointer currentPC) { TargetNUInt relativeAddress = new TargetNUInt(currentPC.Value - mapBase.Value); DecomposeAddress(relativeAddress, out ulong mapIdx, out uint bucketByteIndex); - MapUnit t = _target.Read(GetMapUnitAddress(mapStart, mapIdx)); + MapUnit t = ReadMapUnit(mapStart, mapIdx); // shift the nibble we want to the least significant position - t = t >>> ComputeNibbleShift(mapIdx); - uint nibble = t & NibbleMask; + t = t.LogicalShiftRight(ComputeNibbleShift(mapIdx)); + uint nibble = t.Nibble; if (nibble != 0 && nibble <= bucketByteIndex) { return GetAbsoluteAddress(mapBase, mapIdx, nibble); @@ -112,25 +130,25 @@ internal TargetPointer FindMethodCode(TargetPointer mapBase, TargetPointer mapSt // search backwards through the current map unit // we processed the lsb nibble, move to the next one - t = NextNibble(t); + t = t.ShiftNextNibble; // if there's any nibble set in the current unit, find it - if (t != 0) + if (!t.IsZero) { mapIdx--; - nibble = t & NibbleMask; + nibble = t.Nibble; while (nibble == 0) { - t = NextNibble(t); + t = t.ShiftNextNibble; mapIdx--; - nibble = t & NibbleMask; + nibble = t.Nibble; } return GetAbsoluteAddress(mapBase, mapIdx, nibble); } // if we were near the beginning of the address space, there is not enough space for the code header, // so we can stop - if (mapIdx < NibblesPerMapUnit) + if (mapIdx < MapUnit.SizeInNibbles) { return TargetPointer.Null; } @@ -138,40 +156,39 @@ internal TargetPointer FindMethodCode(TargetPointer mapBase, TargetPointer mapSt // We're now done with the current map index. // Align the map index and move to the previous map unit, then move back one nibble. #pragma warning disable IDE0054 // use compound assignment - mapIdx = mapIdx & (~(NibblesPerMapUnit - 1)); + mapIdx = mapIdx & (~(MapUnit.SizeInNibbles - 1)); mapIdx--; #pragma warning restore IDE0054 // use compound assignment // read the map unit containing mapIdx and skip over it if it is all zeros - while (mapIdx >= NibblesPerMapUnit) + while (mapIdx >= MapUnit.SizeInNibbles) { - t = _target.Read(GetMapUnitAddress(mapStart, mapIdx)); - if (t != 0) + t = ReadMapUnit(mapStart, mapIdx); + if (!t.IsZero) break; - mapIdx -= NibblesPerMapUnit; + mapIdx -= MapUnit.SizeInNibbles; } // if we went all the way to the front, we didn't find a code header - if (mapIdx < NibblesPerMapUnit) + if (mapIdx < MapUnit.SizeInNibbles) { return TargetPointer.Null; } // move to the correct nibble in the map unit - while (mapIdx != 0 && (t & NibbleMask) == 0) + while (mapIdx != 0 && t.Nibble == 0) { - t = NextNibble(t); + t = t.ShiftNextNibble; mapIdx--; } - if (mapIdx == 0 && t == 0) + if (mapIdx == 0 && t.IsZero) { return TargetPointer.Null; } - nibble = t & NibbleMask; + nibble = t.Nibble; return GetAbsoluteAddress(mapBase, mapIdx, nibble); } } -#pragma warning restore SA1121 // Use built in alias From c3f51db9ba8f1f90d8aed0b21dd05d8bc319933a Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Mon, 7 Oct 2024 11:28:09 -0400 Subject: [PATCH 08/10] NibbleMap: use types --- .../ExecutionManagerHelpers/NibbleMap.cs | 180 ++++++++++++------ .../cdacreader/tests/NibbleMapTests.cs | 4 +- 2 files changed, 120 insertions(+), 64 deletions(-) diff --git a/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/ExecutionManagerHelpers/NibbleMap.cs b/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/ExecutionManagerHelpers/NibbleMap.cs index 4de19ab27f5c7d..b5f7cf6b37679a 100644 --- a/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/ExecutionManagerHelpers/NibbleMap.cs +++ b/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/ExecutionManagerHelpers/NibbleMap.cs @@ -2,6 +2,8 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Numerics; +using System.Diagnostics; +using System; namespace Microsoft.Diagnostics.DataContractReader.ExecutionManagerHelpers; @@ -20,7 +22,8 @@ namespace Microsoft.Diagnostics.DataContractReader.ExecutionManagerHelpers; // units until we find a non-zero nibble. internal sealed class NibbleMap { - // The contents of the map are 32-bit integers, where each nibble is a 4-bit integer. + // We load the map contents as 32-bit integers, which contains 8 4-bit nibbles. + // The algorithm will focus on each nibble in a map unit before moving on to the previous map unit internal readonly struct MapUnit { public const int SizeInBytes = sizeof(uint); @@ -29,17 +32,90 @@ internal readonly struct MapUnit public MapUnit(uint value) => Value = value; - public override string ToString() => Value.ToString(); - - public MapUnit LogicalShiftRight(int bits) => new MapUnit(Value >> bits); + public override string ToString() => $"0x{Value:x}"; // Shift the next nibble into the least significant position. public MapUnit ShiftNextNibble => new MapUnit(Value >>> 4); public const uint NibbleMask = 0x0Fu; - public uint Nibble => Value & NibbleMask; + internal Nibble Nibble => new ((int)(Value & NibbleMask)); public bool IsZero => Value == 0; + + // Assuming mapIdx is the index of a nibble within the current map unit, + // shift the unit so that nibble is in the least significant position and return the result. + public MapUnit FocusOnIndexedNibble(MapKey mapIdx) + { + int shift = mapIdx.GetNibbleShift(); + return new MapUnit (Value >>> shift); + } + } + + // Each nibble is a 4-bit integer that gives a 5-bit offset within a bucket. + // We reserse 0 to mean that there is no method starting at any offset within a bucket + internal readonly struct Nibble + { + public readonly int Value; + + public Nibble(int value) + { + Debug.Assert (value >= 0 && value <= 0xF); + Value = value; + } + + public static Nibble Zero => new Nibble(0); + public bool IsEmpty => Value == 0; + + public ulong TargetByteOffset + { + get + { + Debug.Assert(Value != 0); + return (uint)(Value - 1) * MapUnit.SizeInBytes; + } + } + } + + // The key to the map is the index of an individual nibble + internal readonly struct MapKey + { + private readonly ulong MapIdx; + public MapKey(ulong mapIdx) => MapIdx = mapIdx; + public override string ToString() => $"0x{MapIdx:x}"; + + // The offset of the address in the target space that this map index represents + public ulong TargetByteOffset => MapIdx * BytesPerBucket; + + // The index of the map unit that contains this map index + public ulong ContainingMapUnitIndex => MapIdx / MapUnit.SizeInNibbles; + + // The offset of the map unit that contains this map index + public ulong ContainingMapUnitByteOffset => ContainingMapUnitIndex * MapUnit.SizeInBytes; + + // The map index is the index of a nibble within the map, this gives the index of that nibble within a map unit. + public int NibbleIndexInMapUnit => (int)(MapIdx & (MapUnit.SizeInNibbles - 1)); + + // go to the previous nibble + public MapKey Prev => new MapKey(MapIdx - 1); + + // to to the previous map unit + public MapKey PrevMapUnit => new MapKey(MapIdx - MapUnit.SizeInNibbles); + + // Get a MapKey that is aligned to the first nibble in the map unit that contains this map index + public MapKey AlignDownToMapUnit() =>new MapKey(MapIdx & (~(MapUnit.SizeInNibbles - 1))); + + // If the map index is less than the size of a map unit, we are before the first code header and + // can stop searching + public bool BeforeFirstCodeHeader => MapIdx < MapUnit.SizeInNibbles; + + public bool IsZero => MapIdx == 0; + + // given the index of a nibble in the map, compute how much we have to shift a MapUnit to put that + // nibble in the least significant position. + internal int GetNibbleShift() + { + return 28 - (NibbleIndexInMapUnit * 4); // bit shift - 4 bits per nibble + } } public static NibbleMap Create(Target target) @@ -63,69 +139,54 @@ private NibbleMap(Target target) private const ulong BytesPerBucket = 8 * MapUnit.SizeInBytes; - // given the index of a nibble in the map, compute how much we have to shift a MapUnit to put that - // nibble in the least significant position. - internal static int ComputeNibbleShift(ulong mapIdx) - { - // the low bits of the nibble index give us how many nibbles we have to shift by. - int nibbleOffsetInMapUnit = (int)(mapIdx & (MapUnit.SizeInNibbles - 1)); - return 28 - (nibbleOffsetInMapUnit * 4); // bit shift - 4 bits per nibble - } - - // Given a map index and a nibble index, compute the byte offset in the address space that they represent - private static ulong ComputeByteOffset(ulong mapIdx, uint nibble) - { - return mapIdx * BytesPerBucket + (nibble - 1) * MapUnit.SizeInBytes; - } + // for tests + internal static int ComputeNibbleShift(MapKey mapIdx) => mapIdx.GetNibbleShift(); - // Given a base address, a map index, and a nibble index, compute the absolute address in memory + // Given a base address, a map index, and a nibble value, compute the absolute address in memory // that the index and nibble point to. - private static TargetPointer GetAbsoluteAddress(TargetPointer baseAddress, ulong mapIdx, uint nibble) + private static TargetPointer GetAbsoluteAddress(TargetPointer baseAddress, MapKey mapIdx, Nibble nibble) { - return baseAddress + ComputeByteOffset(mapIdx, nibble); + return baseAddress + mapIdx.TargetByteOffset + nibble.TargetByteOffset; } // Given a relative address, decompose it into // the bucket index and an offset within the bucket. - private static void DecomposeAddress(TargetNUInt relative, out ulong mapIdx, out uint bucketByteIndex) + private static void DecomposeAddress(TargetNUInt relative, out MapKey mapIdx, out Nibble bucketByteIndex) { - mapIdx = relative.Value / BytesPerBucket; - bucketByteIndex = ((uint)(relative.Value & (BytesPerBucket - 1)) / MapUnit.SizeInBytes) + 1; - System.Diagnostics.Debug.Assert(bucketByteIndex == (bucketByteIndex & MapUnit.NibbleMask)); - } - - // Given a logical index into the map, compute the address in memory where that map unit is located - private static TargetPointer GetMapUnitAddress(TargetPointer mapStart, ulong mapIdx) - { - ulong bucketIndex = mapIdx / MapUnit.SizeInNibbles; - return mapStart + bucketIndex * MapUnit.SizeInBytes; + mapIdx = new (relative.Value / BytesPerBucket); + int bucketByteOffset = (int)(relative.Value & (BytesPerBucket - 1)); + bucketByteIndex = new Nibble ((bucketByteOffset / MapUnit.SizeInBytes) + 1); } internal static TargetPointer RoundTripAddress(TargetPointer mapBase, TargetPointer currentPC) { TargetNUInt relativeAddress = new TargetNUInt(currentPC.Value - mapBase.Value); - DecomposeAddress(relativeAddress, out ulong mapIdx, out uint bucketByteIndex); + DecomposeAddress(relativeAddress, out MapKey mapIdx, out Nibble bucketByteIndex); return GetAbsoluteAddress(mapBase, mapIdx, bucketByteIndex); } - private MapUnit ReadMapUnit(TargetPointer mapStart, ulong mapIdx) + private MapUnit ReadMapUnit(TargetPointer mapStart, MapKey mapIdx) { - return new MapUnit (_target.Read(GetMapUnitAddress(mapStart, mapIdx))); + // Given a logical index into the map, compute the address in memory where that map unit is located + TargetPointer mapUnitAdderss = mapStart + mapIdx.ContainingMapUnitByteOffset; + return new MapUnit (_target.Read(mapUnitAdderss)); } internal TargetPointer FindMethodCode(TargetPointer mapBase, TargetPointer mapStart, TargetCodePointer currentPC) { TargetNUInt relativeAddress = new TargetNUInt(currentPC.Value - mapBase.Value); - DecomposeAddress(relativeAddress, out ulong mapIdx, out uint bucketByteIndex); + DecomposeAddress(relativeAddress, out MapKey mapIdx, out Nibble bucketByteIndex); MapUnit t = ReadMapUnit(mapStart, mapIdx); // shift the nibble we want to the least significant position - t = t.LogicalShiftRight(ComputeNibbleShift(mapIdx)); - uint nibble = t.Nibble; - if (nibble != 0 && nibble <= bucketByteIndex) + t = t.FocusOnIndexedNibble(mapIdx); + + // if the nibble is non-zero, we have found the start of a method, + // but we need to check that the start is before the current address, not after + if (!t.Nibble.IsEmpty && t.Nibble.Value <= bucketByteIndex.Value) { - return GetAbsoluteAddress(mapBase, mapIdx, nibble); + return GetAbsoluteAddress(mapBase, mapIdx, t.Nibble); } // search backwards through the current map unit @@ -135,60 +196,55 @@ internal TargetPointer FindMethodCode(TargetPointer mapBase, TargetPointer mapSt // if there's any nibble set in the current unit, find it if (!t.IsZero) { - mapIdx--; - nibble = t.Nibble; - while (nibble == 0) + mapIdx = mapIdx.Prev; + while (t.Nibble.IsEmpty) { t = t.ShiftNextNibble; - mapIdx--; - nibble = t.Nibble; + mapIdx = mapIdx.Prev; } - return GetAbsoluteAddress(mapBase, mapIdx, nibble); + return GetAbsoluteAddress(mapBase, mapIdx, t.Nibble); } // if we were near the beginning of the address space, there is not enough space for the code header, // so we can stop - if (mapIdx < MapUnit.SizeInNibbles) + if (mapIdx.BeforeFirstCodeHeader) { return TargetPointer.Null; } - // We're now done with the current map index. - // Align the map index and move to the previous map unit, then move back one nibble. -#pragma warning disable IDE0054 // use compound assignment - mapIdx = mapIdx & (~(MapUnit.SizeInNibbles - 1)); - mapIdx--; -#pragma warning restore IDE0054 // use compound assignment + // We're now done with the current map unit. + // Align the map index to the current map unit, then move back one nibble into the previous map unit + mapIdx = mapIdx.AlignDownToMapUnit(); + mapIdx = mapIdx.Prev; // read the map unit containing mapIdx and skip over it if it is all zeros - while (mapIdx >= MapUnit.SizeInNibbles) + while (!mapIdx.BeforeFirstCodeHeader) { t = ReadMapUnit(mapStart, mapIdx); if (!t.IsZero) break; - mapIdx -= MapUnit.SizeInNibbles; + mapIdx = mapIdx.PrevMapUnit; } // if we went all the way to the front, we didn't find a code header - if (mapIdx < MapUnit.SizeInNibbles) + if (mapIdx.BeforeFirstCodeHeader) { return TargetPointer.Null; } // move to the correct nibble in the map unit - while (mapIdx != 0 && t.Nibble == 0) + while (!mapIdx.IsZero && t.Nibble.IsEmpty) { t = t.ShiftNextNibble; - mapIdx--; + mapIdx = mapIdx.Prev; } - if (mapIdx == 0 && t.IsZero) + if (mapIdx.IsZero && t.IsZero) { return TargetPointer.Null; } - nibble = t.Nibble; - return GetAbsoluteAddress(mapBase, mapIdx, nibble); + return GetAbsoluteAddress(mapBase, mapIdx, t.Nibble); } } diff --git a/src/native/managed/cdacreader/tests/NibbleMapTests.cs b/src/native/managed/cdacreader/tests/NibbleMapTests.cs index 666202bc4c492b..4137fdf8154c04 100644 --- a/src/native/managed/cdacreader/tests/NibbleMapTests.cs +++ b/src/native/managed/cdacreader/tests/NibbleMapTests.cs @@ -140,9 +140,9 @@ public void ExhaustiveNibbbleShifts(ulong irrelevant) int expectedShift = 28; for (int i = 0; i < 255; i++) { - ulong input = irrelevant + (ulong)i; + NibbleMap.MapKey input = new (irrelevant + (ulong)i); int actualShift = NibbleMap.ComputeNibbleShift(input); - Assert.True(expectedShift == actualShift, $"Expected {expectedShift}, got {actualShift} for input 0x{input:x}"); + Assert.True(expectedShift == actualShift, $"Expected {expectedShift}, got {actualShift} for input {input}"); expectedShift -= 4; if (expectedShift == -4) { From 599d533dcf571869d7e5e84bcf8ceba08faf18cb Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Mon, 7 Oct 2024 13:52:02 -0400 Subject: [PATCH 09/10] Clarify termination condition; clean up docs Add an example --- docs/design/datacontracts/ExecutionManager.md | 48 +++++++++++++++ .../ExecutionManagerHelpers/NibbleMap.cs | 58 +++++++++++++++---- 2 files changed, 94 insertions(+), 12 deletions(-) diff --git a/docs/design/datacontracts/ExecutionManager.md b/docs/design/datacontracts/ExecutionManager.md index eb986235463b3d..bd44a68482fab5 100644 --- a/docs/design/datacontracts/ExecutionManager.md +++ b/docs/design/datacontracts/ExecutionManager.md @@ -19,3 +19,51 @@ that allows mapping of a code address in a contiguous subsection of the address space to the pointer to the start of that a code sequence. It takes advantage of the fact that the code starts are aligned and are spaced apart to represent their addresses as a 4-bit nibble value. + +Given a contiguous region of memory in which we lay out a collection of non-overlapping code blocks that are +not too small (so that two adjacent ones aren't too close together) and where the start of each code block is preceeded by a code header aligned on some power of 2, +we can break up the whole memory space into buckets of a fixed size (32-bytes in the current implementation), where +each bucket either has a code block header or not. +Thinking of each code block header address as a hex number, we can view it as: `[index, offset, zeros]` +where each index gives us a bucket and the offset gives us the position of the header within the bucket. +We encode each offset into a 4-bit nibble, reserving the special value 0 to mark the places in the map where a method doesn't start. + +To find the start of a method given an address we first convert it into a bucket index (giving the map unit) +and an offset which we can then turn into the index of the nibble that covers that address. +If the nibble is non-zero, we have the start of a method and it is near the given address. +If the nibble is zero, we have to search backward first through the current map unit, and then through previous map +units until we find a non-zero nibble. + +For example (all code addresses are relative to some unspecified base): + +Suppose there is code starting at address 304 (0x130) + +* Then the map index will be 304 / 32 = 9 and the byte offset will be 304 % 32 = 16 +* Because addresses are 4-byte aligned, the nibble value will be 1 + 16 / 4 = 5 (we reserve 0 to mean no method). +* So the map unit containing index 9 will contain the value 0x5 << 22 (the map index 2 means we want the second nibble in the second map unit, and we number the nibbles starting from the most significant) , or 0x1400000 + + +Now suppose we do a lookup for address 306 (0x132) +* The map index will be 306 / 32 = 9 and the byte offset will be 306 % 32 = 18 +* The nibble value will be 1 + 18 / 4 = 5 +* To do the lookup, we will load the map unit with index 9 (so the second 32-bit unit in the map) and get the value 0x1400000 +* We will then shift to focus on the nibble with map index 9 (which again has nibble shift 22), so + the map unit will be 0x00000005 and we will get the nibble value 5. +* Therefore we know that there is a method start at map index 9, nibble value 5. +* The map index corresponds to an offset of 288 bytes and the nibble value 5 corresponds to an offset of (5 - 1) * 4 = 16 bytes +* So the method starts at offset 288 + 16 = 304, which is the address we were looking for. + +Now suppose we do a lookup for address 302 (0x12E) + +* The map index will be 302 / 32 = 9 and the byte offset will be 302 % 32 = 14 +* The nibble value will be 1 + 14 / 4 = 4 +* To do the lookup, we will load the map unit containing map index 9 and get the value 0x1400000 +* We will then shift to focus on the nibble with map index 9 (which again has nibble shift 22), so we will get + the nibble value 5. +* Therefore we know that there is a method start at map index 9, nibble value 5. +* But the address we're looking for is map index 9, nibble value 4. +* We know that methods can't start within 32-bytes of each other, so we know that the method we're looking for is not in the current nibble. +* We will then try to shift to the previous nibble in the map unit (0x00000005 >> 4 = 0x00000000) +* Therefore we know there is no method start at any map index in the current map unit. +* We will then align the map index to the start of the current map unit (map index 8) and move back to the previous map unit (map index 7) +* At that point, we scan backwards for a non-zero map unit and a non-zero nibble within the first non-zero map unit. Since there are none, we return null. diff --git a/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/ExecutionManagerHelpers/NibbleMap.cs b/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/ExecutionManagerHelpers/NibbleMap.cs index b5f7cf6b37679a..a4f28a584c5df4 100644 --- a/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/ExecutionManagerHelpers/NibbleMap.cs +++ b/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/ExecutionManagerHelpers/NibbleMap.cs @@ -20,7 +20,39 @@ namespace Microsoft.Diagnostics.DataContractReader.ExecutionManagerHelpers; // If the nibble is non-zero, we have the start of a method and it is near the given address. // If the nibble is zero, we have to search backward first through the current map unit, and then through previous map // units until we find a non-zero nibble. -internal sealed class NibbleMap +// +// For example (all code addresses are relative to some unspecified base): +// Suppose there is code starting at address 304 (0x130) +// Then the map index will be 304 / 32 = 9 and the byte offset will be 304 % 32 = 16 +// Because addresses are 4-byte aligned, the nibble value will be 1 + 16 / 4 = 5 (we reserve 0 to mean no method). +// So the map unit containing index 9 will contain the value 0x5 << 22 (the map index 2 means we want the second nibble in the second map unit, and we number the nibbles starting from the most significant) +// Or 0x1400000 +// +// Now suppose we do a lookup for address 306 (0x132) +// The map index will be 306 / 32 = 9 and the byte offset will be 306 % 32 = 18 +// The nibble value will be 1 + 18 / 4 = 5 +// To do the lookup, we will load the map unit with index 9 (so the second 32-bit unit in the map) and get the value 0x1400000 +// We will then shift to focus on the nibble with map index 9 (which again has nibble shift 22), so +// the map unit will be 0x00000005 and we will get the nibble value 5. +// Therefore we know that there is a method start at map index 9, nibble value 5. +// The map index corresponds to an offset of 288 bytes and the nibble value 5 corresponds to an offset of (5 - 1) * 4 = 16 bytes +// So the method starts at offset 288 + 16 = 304, which is the address we were looking for. +// +// Now suppose we do a lookup for address 302 (0x12E) +// The map index will be 302 / 32 = 9 and the byte offset will be 302 % 32 = 14 +// The nibble value will be 1 + 14 / 4 = 4 +// To do the lookup, we will load the map unit containing map index 9 and get the value 0x1400000 +// We will then shift to focus on the nibble with map index 9 (which again has nibble shift 22), so we will get +// the nibble value 5. +// Therefore we know that there is a method start at map index 9, nibble value 5. +// But the address we're looking for is map index 9, nibble value 4. +// We know that methods can't start within 32-bytes of each other, so we know that the method we're looking for is not in the current nibble. +// We will then try to shift to the previous nibble in the map unit (0x00000005 >> 4 = 0x00000000) +// Therefore we know there is no method start at any map index in the current map unit. +// We will then align the map index to the start of the current map unit (map index 8) and move back to the previous map unit (map index 7) +// At that point, we scan backwards for non-zero map units. Since there are none, we return null. + +internal class NibbleMap { // We load the map contents as 32-bit integers, which contains 8 4-bit nibbles. // The algorithm will focus on each nibble in a map unit before moving on to the previous map unit @@ -104,9 +136,9 @@ internal readonly struct MapKey // Get a MapKey that is aligned to the first nibble in the map unit that contains this map index public MapKey AlignDownToMapUnit() =>new MapKey(MapIdx & (~(MapUnit.SizeInNibbles - 1))); - // If the map index is less than the size of a map unit, we are before the first code header and + // If the map index is less than the size of a map unit, we are in the first MapUnit and // can stop searching - public bool BeforeFirstCodeHeader => MapIdx < MapUnit.SizeInNibbles; + public bool InFirstMapUnit => MapIdx < MapUnit.SizeInNibbles; public bool IsZero => MapIdx == 0; @@ -205,9 +237,9 @@ internal TargetPointer FindMethodCode(TargetPointer mapBase, TargetPointer mapSt return GetAbsoluteAddress(mapBase, mapIdx, t.Nibble); } - // if we were near the beginning of the address space, there is not enough space for the code header, - // so we can stop - if (mapIdx.BeforeFirstCodeHeader) + // We finished the current map unit, we want to move to the previous one. + // But if we were in the first map unit, we can stop + if (mapIdx.InFirstMapUnit) { return TargetPointer.Null; } @@ -218,19 +250,21 @@ internal TargetPointer FindMethodCode(TargetPointer mapBase, TargetPointer mapSt mapIdx = mapIdx.Prev; // read the map unit containing mapIdx and skip over it if it is all zeros - while (!mapIdx.BeforeFirstCodeHeader) + while (true) { t = ReadMapUnit(mapStart, mapIdx); if (!t.IsZero) break; + if (mapIdx.InFirstMapUnit) + { + // we're at the first map unit and all the bits in the map unit are zero, + // there is no code header to find + return TargetPointer.Null; + } mapIdx = mapIdx.PrevMapUnit; } - // if we went all the way to the front, we didn't find a code header - if (mapIdx.BeforeFirstCodeHeader) - { - return TargetPointer.Null; - } + Debug.Assert(!t.IsZero); // move to the correct nibble in the map unit while (!mapIdx.IsZero && t.Nibble.IsEmpty) From 4182125dae4769d5745bb8d21831dffa4df0ebd1 Mon Sep 17 00:00:00 2001 From: Aleksey Kliger Date: Mon, 7 Oct 2024 19:36:43 -0400 Subject: [PATCH 10/10] fix dodgy math in example --- docs/design/datacontracts/ExecutionManager.md | 9 +++++---- .../ExecutionManagerHelpers/NibbleMap.cs | 14 +++++++------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/docs/design/datacontracts/ExecutionManager.md b/docs/design/datacontracts/ExecutionManager.md index bd44a68482fab5..76f8a3b4dd295e 100644 --- a/docs/design/datacontracts/ExecutionManager.md +++ b/docs/design/datacontracts/ExecutionManager.md @@ -40,14 +40,15 @@ Suppose there is code starting at address 304 (0x130) * Then the map index will be 304 / 32 = 9 and the byte offset will be 304 % 32 = 16 * Because addresses are 4-byte aligned, the nibble value will be 1 + 16 / 4 = 5 (we reserve 0 to mean no method). -* So the map unit containing index 9 will contain the value 0x5 << 22 (the map index 2 means we want the second nibble in the second map unit, and we number the nibbles starting from the most significant) , or 0x1400000 +* So the map unit containing index 9 will contain the value 0x5 << 24 (the map index 9 means we want the second nibble in the second map unit, and we number the nibbles starting from the most significant) , or +0x05000000 Now suppose we do a lookup for address 306 (0x132) * The map index will be 306 / 32 = 9 and the byte offset will be 306 % 32 = 18 * The nibble value will be 1 + 18 / 4 = 5 -* To do the lookup, we will load the map unit with index 9 (so the second 32-bit unit in the map) and get the value 0x1400000 -* We will then shift to focus on the nibble with map index 9 (which again has nibble shift 22), so +* To do the lookup, we will load the map unit with index 9 (so the second 32-bit unit in the map) and get the value 0x05000000 +* We will then shift to focus on the nibble with map index 9 (which again has nibble shift 24), so the map unit will be 0x00000005 and we will get the nibble value 5. * Therefore we know that there is a method start at map index 9, nibble value 5. * The map index corresponds to an offset of 288 bytes and the nibble value 5 corresponds to an offset of (5 - 1) * 4 = 16 bytes @@ -57,7 +58,7 @@ Now suppose we do a lookup for address 302 (0x12E) * The map index will be 302 / 32 = 9 and the byte offset will be 302 % 32 = 14 * The nibble value will be 1 + 14 / 4 = 4 -* To do the lookup, we will load the map unit containing map index 9 and get the value 0x1400000 +* To do the lookup, we will load the map unit containing map index 9 and get the value 0x05000000 * We will then shift to focus on the nibble with map index 9 (which again has nibble shift 22), so we will get the nibble value 5. * Therefore we know that there is a method start at map index 9, nibble value 5. diff --git a/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/ExecutionManagerHelpers/NibbleMap.cs b/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/ExecutionManagerHelpers/NibbleMap.cs index a4f28a584c5df4..3916b4db680193 100644 --- a/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/ExecutionManagerHelpers/NibbleMap.cs +++ b/src/native/managed/cdacreader/Microsoft.Diagnostics.DataContractReader.Contracts/ExecutionManagerHelpers/NibbleMap.cs @@ -25,14 +25,14 @@ namespace Microsoft.Diagnostics.DataContractReader.ExecutionManagerHelpers; // Suppose there is code starting at address 304 (0x130) // Then the map index will be 304 / 32 = 9 and the byte offset will be 304 % 32 = 16 // Because addresses are 4-byte aligned, the nibble value will be 1 + 16 / 4 = 5 (we reserve 0 to mean no method). -// So the map unit containing index 9 will contain the value 0x5 << 22 (the map index 2 means we want the second nibble in the second map unit, and we number the nibbles starting from the most significant) -// Or 0x1400000 +// So the map unit containing index 9 will contain the value 0x5 << 24 (the map index 9 means we want the second nibble in the second map unit, and we number the nibbles starting from the most significant) +// Or 0x05000000 // // Now suppose we do a lookup for address 306 (0x132) // The map index will be 306 / 32 = 9 and the byte offset will be 306 % 32 = 18 // The nibble value will be 1 + 18 / 4 = 5 -// To do the lookup, we will load the map unit with index 9 (so the second 32-bit unit in the map) and get the value 0x1400000 -// We will then shift to focus on the nibble with map index 9 (which again has nibble shift 22), so +// To do the lookup, we will load the map unit with index 9 (so the second 32-bit unit in the map) and get the value 0x05000000 +// We will then shift to focus on the nibble with map index 9 (which again has nibble shift 24), so // the map unit will be 0x00000005 and we will get the nibble value 5. // Therefore we know that there is a method start at map index 9, nibble value 5. // The map index corresponds to an offset of 288 bytes and the nibble value 5 corresponds to an offset of (5 - 1) * 4 = 16 bytes @@ -41,8 +41,8 @@ namespace Microsoft.Diagnostics.DataContractReader.ExecutionManagerHelpers; // Now suppose we do a lookup for address 302 (0x12E) // The map index will be 302 / 32 = 9 and the byte offset will be 302 % 32 = 14 // The nibble value will be 1 + 14 / 4 = 4 -// To do the lookup, we will load the map unit containing map index 9 and get the value 0x1400000 -// We will then shift to focus on the nibble with map index 9 (which again has nibble shift 22), so we will get +// To do the lookup, we will load the map unit containing map index 9 and get the value 0x05000000 +// We will then shift to focus on the nibble with map index 9 (which again has nibble shift 24), so we will get // the nibble value 5. // Therefore we know that there is a method start at map index 9, nibble value 5. // But the address we're looking for is map index 9, nibble value 4. @@ -83,7 +83,7 @@ public MapUnit FocusOnIndexedNibble(MapKey mapIdx) } } - // Each nibble is a 4-bit integer that gives a 5-bit offset within a bucket. + // Each nibble is a 4-bit integer that gives an offset within a bucket. // We reserse 0 to mean that there is no method starting at any offset within a bucket internal readonly struct Nibble {