Load XML from EventLogReader (#310)

With this change, we load the XML from the EventLogReader by calling
ToXml() on the EventRecord, rather than generating our own XML from the
Properties collection.

Pros:

* Full-fidelity XML instead of the partial XML we had before.

Cons:

* Loading the XML this way takes time. We do this in the background
after the initial load, and the status is reflected in the status bar.
Note the user can navigate to events and see XML immediately (we resolve
it when they click on the event) even when this job is in progress.
* Storing these big XML strings take memory. With a set of 4 logs that
caused EventLogExpert to use about 7 GB of memory before this change, we
now use about 10.5 GB of memory for the same logs.

src/EventLogExpert.Eventing/EventResolvers/EventProviderDatabaseEventResolver.cs

@@ -205,20 +205,18 @@ public DisplayEventModel Resolve(EventRecord eventRecord, string OwningLogName)
                 eventRecord.ProviderName,
                 "",
                 "Description not found. No provider available.",
-                eventProperties,
                 eventRecord.Qualifiers,
-                eventRecord.Keywords,
                 GetKeywordsFromBitmask(eventRecord.Keywords, null),
                 eventRecord.ProcessId,
                 eventRecord.ThreadId,
                 eventRecord.LogName,
-                null,
-                OwningLogName);
+                OwningLogName,
+                eventRecord);
         }
 
         if (lastResult.Description == null)
         {
-            lastResult = lastResult with { Description = "" };
+            lastResult.Description = "";
         }
 
         return lastResult;

src/EventLogExpert.Eventing/EventResolvers/EventReaderEventResolver.cs

@@ -44,15 +44,13 @@ public DisplayEventModel Resolve(EventRecord eventRecord, string OwningLogName)
             eventRecord.ProviderName,
             eventRecord.Task is 0 or null ? "None" : TryGetValue(() => eventRecord.TaskDisplayName),
             string.IsNullOrEmpty(desc) ? string.Empty : desc,
-            eventRecord.Properties,
             eventRecord.Qualifiers,
-            eventRecord.Keywords,
             keywordsDisplayNames,
             eventRecord.ProcessId,
             eventRecord.ThreadId,
             eventRecord.LogName,
-            null,
-            OwningLogName);
+            OwningLogName,
+            eventRecord);
     }
 
     private static T TryGetValue<T>(Func<T> func)

src/EventLogExpert.Eventing/EventResolvers/EventResolverBase.cs

@@ -295,15 +295,13 @@ protected DisplayEventModel ResolveFromProviderDetails(
             eventRecord.ProviderName,
             taskName?.TrimEnd('\0') ?? string.Empty,
             description?.TrimEnd('\0') ?? "Unable to format description",
-            eventProperties,
             eventRecord.Qualifiers,
-            eventRecord.Keywords,
             GetKeywordsFromBitmask(eventRecord.Keywords, providerDetails),
             eventRecord.ProcessId,
             eventRecord.ThreadId,
             eventRecord.LogName!,
-            template,
-            owningLogName);
+            owningLogName,
+            eventRecord);
     }
 
     [GeneratedRegex("%+[0-9]+")]

src/EventLogExpert.Eventing/EventResolvers/LocalProviderEventResolver.cs

@@ -51,15 +51,13 @@ public DisplayEventModel Resolve(EventRecord eventRecord, string OwningLogName)
                 eventRecord.ProviderName,
                 "",
                 "Description not found. No provider available.",
-                eventRecord.Properties,
                 eventRecord.Qualifiers,
-                eventRecord.Keywords,
                 GetKeywordsFromBitmask(eventRecord.Keywords, null),
                 eventRecord.ProcessId,
                 eventRecord.ThreadId,
                 eventRecord.LogName,
-                null,
-                OwningLogName);
+                OwningLogName,
+                eventRecord);
         }
 
         // The Properties getter is expensive, so we only call the getter once,
@@ -70,7 +68,7 @@ public DisplayEventModel Resolve(EventRecord eventRecord, string OwningLogName)
 
         if (result.Description == null)
         {
-            result = result with { Description = "" };
+            result.Description = "";
         }
 
         return result;

src/EventLogExpert.Eventing/Models/DisplayEventModel.cs

@@ -2,120 +2,105 @@
 // // Licensed under the MIT License.
 
 using System.Diagnostics.Eventing.Reader;
-using System.Text;
 
 namespace EventLogExpert.Eventing.Models;
 
-public sealed record DisplayEventModel(
-    long? RecordId,
-    Guid? ActivityId,
-    DateTime TimeCreated,
-    int Id,
-    string ComputerName,
-    string Level,
-    string Source,
-    string TaskCategory,
-    string Description,
-    IList<EventProperty> Properties,
-    int? Qualifiers,
-    long? Keywords,
-    IEnumerable<string> KeywordsDisplayNames,
-    int? ProcessId,
-    int? ThreadId,
-    string LogName, // This is the log name from the event reader
-    string? Template,
-    string OwningLog) // This is the name of the log file or the live log, which we use internally
+public sealed class DisplayEventModel
 {
-    public string Xml
-    {
-        get
-        {
-            StringBuilder sb = new();
+    public Guid? ActivityId { get; }
+    public string ComputerName { get; }
+    public string Description { get; set; }
+    public int Id { get; }
+    public IEnumerable<string> KeywordsDisplayNames { get; }
+    public string Level { get; }
+    public string LogName { get; }
+    public string OwningLog { get; }
+    public int? ProcessId { get; }
+    public int? Qualifiers { get; }
+    public long? RecordId { get; }
+    public string Source { get; }
+    public string TaskCategory { get; }
+    public int? ThreadId { get; }
+    public DateTime TimeCreated { get; }
 
-            sb.AppendLine($"""
-            <Event xmlns="http://schemas.microsoft.com/win/2004/08/events/event">
-                <System>
-                    <Provider Name="{Source}" />
-                    <EventID{(Qualifiers.HasValue ? $" Qualifiers=\"{Qualifiers.Value}\"" : "")}>{Id}</EventID>
-                    <Level>{Level}</Level>
-                    <Task>{TaskCategory}</Task>
-                    <Keywords>{(Keywords.HasValue ? "0x" + Keywords.Value.ToString("X") : "0x0")}</Keywords>
-                    <TimeCreated SystemTime="{TimeCreated.ToUniversalTime():o}" />
-                    <EventRecordID>{RecordId}</EventRecordID>
-                    {(ActivityId is null ? "<Correlation />" : $"<Correlation ActivityID=\"{ActivityId}\" />")}
-                    {(ProcessId is null && ThreadId is null ?
-                        "<Execution />" :
-                        $"<Execution ProcessID=\"{ProcessId}\" ThreadID=\"{ThreadId}\" />")}
-                    <Channel>{LogName}</Channel>
-                    <Computer>{ComputerName}</Computer>
-                </System>
-                <EventData>
-            """);
-
-            sb.Append(GetEventData());
+    public DisplayEventModel(
+    long? recordId,
+    Guid? activityId,
+    DateTime timeCreated,
+    int id,
+    string computerName,
+    string level,
+    string source,
+    string taskCategory,
+    string description,
+    int? qualifiers,
+    IEnumerable<string> keywordsDisplayNames,
+    int? processId,
+    int? threadId,
+    string logName, // This is the log name from the event reader
+    string owningLog, // This is the name of the log file or the live log, which we use internally
+    EventRecord eventRecord)
+    {
+        // Public immutable properties
+        ActivityId = activityId;
+        ComputerName = computerName;
+        Id = id;
+        KeywordsDisplayNames = keywordsDisplayNames;
+        Level = level;
+        LogName = logName;
+        OwningLog = owningLog;
+        ProcessId = processId;
+        Qualifiers = qualifiers;
+        RecordId = recordId;
+        Source = source;
+        TaskCategory = taskCategory;
+        ThreadId = threadId;
+        TimeCreated = timeCreated;
 
-            sb.Append("""
-                </EventData>
-            </Event>
-            """);
+        // Public mutable properties
+        Description = description;
 
-            return sb.ToString();
-        }
+        // Private properties
+        _eventRecord = eventRecord;
     }
 
-    private string GetEventData()
+    public string Xml
     {
-        StringBuilder sb = new();
-
-        if (!string.IsNullOrEmpty(Template))
+        get
         {
-            try
+            if (_cachedXml is not null)
             {
-                List<string> propertyNames = [];
-                int index = -1;
-
-                while (-1 < (index = Template.IndexOf("name=", index + 1, StringComparison.Ordinal)))
-                {
-                    var nameStart = index + 6;
-                    var nameEnd = Template.IndexOf('"', nameStart);
-                    var name = Template[nameStart..nameEnd];
-                    propertyNames.Add(name);
-                }
-
-                for (var i = 0; i < Properties.Count; i++)
+                return _cachedXml;
+            }
+
+            lock (this)
+            {
+                if (_cachedXml is null)
                 {
-                    if (i >= propertyNames.Count) { break; }
+                    if (_eventRecord is null)
+                    {
+                        return "Unable to get XML. EventRecord is null.";
+                    }
 
-                    if (Properties[i].Value is byte[] val)
+                    var unformattedXml = _eventRecord.ToXml();
+                    try
                     {
-                        sb.AppendLine($"           <Data Name=\"{propertyNames[i]}\">{Convert.ToHexString(val)}</Data>");
+                        _cachedXml = System.Xml.Linq.XElement.Parse(unformattedXml).ToString();
                     }
-                    else
+                    catch
                     {
-                        sb.AppendLine($"           <Data Name=\"{propertyNames[i]}\">{Properties[i].Value}</Data>");
+                        _cachedXml = unformattedXml;
                     }
+
+                    _eventRecord = null;
                 }
 
-                return sb.ToString();
-            }
-            catch
-            {
-                // No tracer available here
+                return _cachedXml;
             }
         }
+    }
 
-        foreach (var p in Properties)
-        {
-            if (p.Value is byte[] bytes)
-            {
-                sb.AppendLine($"            <Data>{Convert.ToHexString(bytes)}</Data>");
-            }
-            else
-            {
-                sb.AppendLine($"            <Data>{p.Value}</Data>");
-            }
-        }
+    private string? _cachedXml = null;
 
-        return sb.ToString();
-    }
+    private EventRecord? _eventRecord;
 }

src/EventLogExpert.UI/Store/EventLog/EventLogAction.cs

@@ -46,5 +46,7 @@ public sealed record SetContinouslyUpdate(bool ContinuouslyUpdate);
     /// <param name="Count"></param>
     public sealed record SetEventsLoading(Guid ActivityId, int Count);
 
+    public sealed record SetXmlLoading(Guid ActivityId, int Count);
+
     public sealed record SetFilters(EventFilter EventFilter);
 }

src/EventLogExpert.UI/Store/EventLog/EventLogEffects.cs

@@ -188,6 +188,22 @@ await Task.Run(() =>
                     {
                         logWatcherService.AddLog(action.LogName, lastEvent?.Bookmark);
                     }
+
+                    sw.Restart();
+                    for (int i = 0; i < events.Count; i++)
+                    {
+                        if (sw.ElapsedMilliseconds > 1000)
+                        {
+                            sw.Restart();
+                            dispatcher.Dispatch(new EventLogAction.SetXmlLoading(activityId, i));
+                        }
+
+                        _ = events[i].Xml;
+                    }
+
+                    dispatcher.Dispatch(new EventLogAction.SetXmlLoading(activityId, 0));
+
+                    dispatcher.Dispatch(new StatusBarAction.SetResolverStatus($""));
                 }
                 finally
                 {

src/EventLogExpert.UI/Store/EventLog/EventLogReducers.cs

@@ -4,6 +4,7 @@
 using EventLogExpert.Eventing.Models;
 using EventLogExpert.UI.Models;
 using Fluxor;
+using System;
 using System.Collections.Immutable;
 
 namespace EventLogExpert.UI.Store.EventLog;
@@ -95,19 +96,7 @@ public static EventLogState ReduceSetContinouslyUpdate(
     [ReducerMethod]
     public static EventLogState ReduceSetEventsLoading(EventLogState state, EventLogAction.SetEventsLoading action)
     {
-        var newEventsLoading = state.EventsLoading;
-
-        if (newEventsLoading.ContainsKey(action.ActivityId))
-        {
-            newEventsLoading = newEventsLoading.Remove(action.ActivityId);
-        }
-
-        if (action.Count == 0)
-        {
-            return state with { EventsLoading = newEventsLoading };
-        }
-
-        return state with { EventsLoading = newEventsLoading.Add(action.ActivityId, action.Count) };
+        return state with { EventsLoading = CommonLoadingReducer(state.EventsLoading, action.ActivityId, action.Count) };
     }
 
     [ReducerMethod]
@@ -121,6 +110,27 @@ public static EventLogState ReduceSetFilters(EventLogState state, EventLogAction
         return state with { AppliedFilter = action.EventFilter };
     }
 
+    [ReducerMethod]
+    public static EventLogState ReduceSetXmlLoading(EventLogState state, EventLogAction.SetXmlLoading action)
+    {
+        return state with { XmlLoading = CommonLoadingReducer(state.XmlLoading, action.ActivityId, action.Count) };
+    }
+
+    private static ImmutableDictionary<Guid, int> CommonLoadingReducer(ImmutableDictionary<Guid, int> loadingEntries, Guid activityId, int count)
+    {
+        if (loadingEntries.ContainsKey(activityId))
+        {
+            loadingEntries = loadingEntries.Remove(activityId);
+        }
+
+        if (count == 0)
+        {
+            return loadingEntries;
+        }
+
+        return loadingEntries.Add(activityId, count);
+    }
+
     private static EventLogData GetEmptyLogData(string logName, LogType logType) => new(
         logName,
         logType,

src/EventLogExpert.UI/Store/EventLog/EventLogState.cs

@@ -30,4 +30,6 @@ public sealed record EventLogState
     public bool NewEventBufferIsFull { get; init; }
 
     public DisplayEventModel? SelectedEvent { get; init; }
+
+    public ImmutableDictionary<Guid, int> XmlLoading { get; init; } = ImmutableDictionary<Guid, int>.Empty;
 }

src/EventLogExpert.UI/Store/EventLog/LiveLogWatcherService.cs

@@ -166,6 +166,7 @@ private void StartWatching(string logName)
                     }
 
                     var resolved = _resolver.Resolve(eventArgs.EventRecord, logName);
+                    _ = resolved.Xml; // Immediately cache the ToXml() result.
                     _dispatcher.Dispatch(new EventLogAction.AddEvent(resolved));
                 }
             };

src/EventLogExpert/Components/StatusBar.razor

@@ -16,6 +16,11 @@
         <span>Loading: @loadingProgress.Value</span>
     }
 
+    @foreach (var xmlProgress in EventLogState.Value.XmlLoading)
+    {
+        <span>Populating XML: @xmlProgress.Value</span>
+    }
+
     <span>Events Loaded: @EventLogState.Value.ActiveLogs.Values.Sum(log => log.Events.Count)</span>
 
     @if (EventTableState.Value.ActiveTableId is not null && FilterMethods.IsFilteringEnabled(EventLogState.Value.AppliedFilter))