Merge pull request #223 from nblumhardt/experimental-unflattening-sup…

…port

Convert dotted property names into nested values when Serilog v4's experimental `AppContext` switch is set

src/Serilog.Sinks.Seq/Sinks/Seq/Conventions/PreserveDottedPropertyNames.cs

@@ -0,0 +1,31 @@
+// Copyright © Serilog Contributors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+using System.Collections.Generic;
+using Serilog.Events;
+
+namespace Serilog.Sinks.Seq.Conventions;
+
+/// <summary>
+/// Maintains verbatim processing of property names. A property named <c>"a.b"</c> will be transmitted to Seq as a
+/// scalar value with name <c>"a.b"</c>.
+/// </summary>
+class PreserveDottedPropertyNames: IDottedPropertyNameConvention
+{
+    /// <inheritdoc />
+    public IReadOnlyDictionary<string, LogEventPropertyValue> ProcessDottedPropertyNames(IReadOnlyDictionary<string, LogEventPropertyValue> maybeDotted)
+    {
+        return maybeDotted;
+    }
+}

src/Serilog.Sinks.Seq/Sinks/Seq/Conventions/UnflattenDottedPropertyNames.cs

@@ -0,0 +1,131 @@
+// Copyright © Serilog Contributors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using Serilog.Events;
+
+namespace Serilog.Sinks.Seq.Conventions;
+
+/// <summary>
+/// Experimental. Unflatten property names. A property with name <c>"a.b"</c> will be transmitted to Seq as
+/// a structure with name <c>"a"</c>, and one member <c>"b"</c>.
+/// </summary>
+/// <remarks>This behavior is enabled when the <c>Serilog.Parsing.MessageTemplateParser.AcceptDottedPropertyNames</c>
+/// <see cref="AppContext"/> switch is set to value <c langword="true"/>.</remarks>
+class UnflattenDottedPropertyNames: IDottedPropertyNameConvention
+{
+    const int MaxDepth = 10;
+
+    /// <inheritdoc />
+    public IReadOnlyDictionary<string, LogEventPropertyValue> ProcessDottedPropertyNames(IReadOnlyDictionary<string, LogEventPropertyValue> maybeDotted)
+    {
+        return DottedToNestedRecursive(maybeDotted, 0);
+    }
+
+    static IReadOnlyDictionary<string, LogEventPropertyValue> DottedToNestedRecursive(IReadOnlyDictionary<string, LogEventPropertyValue> maybeDotted, int depth)
+    {
+        if (depth == MaxDepth)
+            return maybeDotted;
+
+        // Assume that the majority of entries will be bare or have unique prefixes.
+        var result = new Dictionary<string, LogEventPropertyValue>(maybeDotted.Count);
+
+        // Sorted for determinism.
+        var dotted = new SortedDictionary<string, LogEventPropertyValue>(StringComparer.Ordinal);
+
+        // First - give priority to bare names, since these would otherwise be claimed by the parents of further nested
+        // layers and we'd have nowhere to put them when resolving conflicts. (Dotted entries that conflict can keep their dotted keys).
+
+        foreach (var kv in maybeDotted)
+        {
+            if (IsDottedIdentifier(kv.Key))
+            {
+                // Stash for processing in the next stage.
+                dotted.Add(kv.Key, kv.Value);
+            }
+            else
+            {
+                result.Add(kv.Key, kv.Value);
+            }
+        }
+
+        // Then - for dotted keys with a prefix not already present in the result, convert to structured data and add to
+        // the result. Any set of dotted names that collide with a preexisting key will be left as-is.
+
+        string? prefix = null;
+        Dictionary<string, LogEventPropertyValue>? nested = null;
+        foreach (var kv in dotted)
+        {
+            var (newPrefix, rem) = TakeFirstIdentifier(kv.Key);
+
+            if (prefix != null && prefix != newPrefix)
+            {
+                result.Add(prefix, MakeStructureValue(DottedToNestedRecursive(nested!, depth + 1)));
+                prefix = null;
+                nested = null;
+            }
+
+            if (nested != null && !nested.ContainsKey(rem))
+            {
+                prefix = newPrefix;
+                nested.Add(rem, kv.Value);
+            }
+            else if (nested == null && !result.ContainsKey(newPrefix))
+            {
+                prefix = newPrefix;
+                nested = new () { { rem, kv.Value } };
+            }
+            else
+            {
+                result.Add(kv.Key, kv.Value);
+            }
+        }
+
+        if (prefix != null)
+        {
+            result[prefix] = MakeStructureValue(DottedToNestedRecursive(nested!, depth + 1));
+        }
+
+        return result;
+    }
+
+    static LogEventPropertyValue MakeStructureValue(IReadOnlyDictionary<string,LogEventPropertyValue> properties)
+    {
+        return new StructureValue(properties.Select(kv => new LogEventProperty(kv.Key, kv.Value)), typeTag: null);
+    }
+
+    internal static bool IsDottedIdentifier(string key) =>
+        key.Contains('.') &&
+        !key.StartsWith(".", StringComparison.Ordinal) &&
+        !key.EndsWith(".", StringComparison.Ordinal) &&
+        key.Split('.').All(IsIdentifier);
+
+    static bool IsIdentifier(string s) => s.Length != 0 &&
+                                          !char.IsDigit(s[0]) &&
+                                          s.All(ch => char.IsLetter(ch) || char.IsDigit(ch) || ch == '_');
+
+    static (string, string) TakeFirstIdentifier(string dottedIdentifier)
+    {
+        // We can do this simplistically because keys in `dotted` conform to `IsDottedName`.
+        Debug.Assert(IsDottedIdentifier(dottedIdentifier));
+
+        var firstDot = dottedIdentifier.IndexOf('.');
+        var prefix = dottedIdentifier.Substring(0, firstDot);
+        var rem = dottedIdentifier.Substring(firstDot + 1);
+        return (prefix, rem);
+    }
+}

src/Serilog.Sinks.Seq/Sinks/Seq/IDottedPropertyNameConvention.cs

@@ -0,0 +1,33 @@
+// Copyright © Serilog Contributors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+using System.Collections.Generic;
+using Serilog.Events;
+
+namespace Serilog.Sinks.Seq;
+
+/// <summary>
+/// Enables switching between the experimental "unflattening" behavior applied to dotted property names, and the
+/// regular verbatim property name handling.
+/// </summary>
+interface IDottedPropertyNameConvention
+{
+    /// <summary>
+    /// Convert the properties in <paramref name="maybeDotted"/> into the form specified by the current property
+    /// name processing convention.
+    /// </summary>
+    /// <param name="maybeDotted">The properties associated with a log event.</param>
+    /// <returns>The processed properties.</returns>
+    IReadOnlyDictionary<string, LogEventPropertyValue> ProcessDottedPropertyNames(IReadOnlyDictionary<string, LogEventPropertyValue> maybeDotted);
+}

src/Serilog.Sinks.Seq/Sinks/Seq/SeqCompactJsonFormatter.cs

@@ -1,4 +1,4 @@
-// Copyright 2016 Serilog Contributors
+// Copyright © Serilog Contributors
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -21,6 +21,8 @@
 using Serilog.Formatting;
 using Serilog.Formatting.Json;
 using Serilog.Parsing;
+using Serilog.Sinks.Seq.Conventions;
+
 // ReSharper disable MemberCanBePrivate.Global
 // ReSharper disable PossibleMultipleEnumeration
 
@@ -33,8 +35,13 @@ namespace Serilog.Sinks.Seq;
 /// implicit SerilogTracing span support.</remarks>
 public class SeqCompactJsonFormatter: ITextFormatter
 {
-    readonly JsonValueFormatter _valueFormatter = new("$type");
+    static readonly IDottedPropertyNameConvention DottedPropertyNameConvention =
+        AppContext.TryGetSwitch("Serilog.Parsing.MessageTemplateParser.AcceptDottedPropertyNames", out var accept) && accept ?
+            new UnflattenDottedPropertyNames() :
+            new PreserveDottedPropertyNames();
 
+    readonly JsonValueFormatter _valueFormatter = new("$type");
+
     /// <summary>
     /// Format the log event into the output. Subsequent events will be newline-delimited.
     /// </summary>
@@ -139,8 +146,9 @@ public static void FormatEvent(LogEvent logEvent, TextWriter output, JsonValueFo
                 output.Write('\"');
             }
         }
-
-        foreach (var property in logEvent.Properties)
+
+        var properties = DottedPropertyNameConvention.ProcessDottedPropertyNames(logEvent.Properties);
+        foreach (var property in properties)
         {
             var name = property.Key;
 

test/Serilog.Sinks.Seq.Tests/Conventions/UnflattenDottedPropertyNamesTests.cs

@@ -0,0 +1,122 @@
+using System.Collections.Generic;
+using System.Linq;
+using Serilog.Events;
+using Serilog.Sinks.Seq.Conventions;
+using Xunit;
+
+namespace Serilog.Sinks.Seq.Tests.Conventions;
+
+public class UnflattenDottedPropertyNamesTests
+{
+    [Fact]
+    public void DottedToNestedWorks()
+    {
+        var someDotted = new Dictionary<string, LogEventPropertyValue>
+        {
+            ["dotnet.ilogger.category"] = new ScalarValue("Test.App"),
+            ["environment.name"] = new ScalarValue("Production"),
+            ["environment.region"] = new ScalarValue("us-west-2"),
+            ["environment.beverage.name"] = new ScalarValue("coffee"),
+            ["environment.domains"] = new StructureValue(
+                [
+                    new LogEventProperty("example.com", new ScalarValue(42)),
+                    new LogEventProperty("datalust.co", new ScalarValue(43))
+                ]),
+            ["scope"] = new StructureValue([]),
+            ["scope.name"] = new ScalarValue("Gerald"),
+            ["vegetable"] = new ScalarValue("Potato"),
+            ["Scope"] = new ScalarValue("Periscope"),
+            [".gitattributes"] = new ScalarValue("Text")
+        };
+
+        var expected = new Dictionary<string, LogEventPropertyValue>
+        {
+            ["dotnet"] = new StructureValue(
+            [
+                new LogEventProperty("ilogger", new StructureValue(
+                [
+                    new LogEventProperty("category", new ScalarValue("Test App"))
+                ]))
+            ]),
+            ["environment"] = new StructureValue(
+            [
+                new LogEventProperty("name", new ScalarValue("Production")),
+                new LogEventProperty("region", new ScalarValue("us-west-2")),
+                new LogEventProperty("beverage", new StructureValue(
+                [
+                    new LogEventProperty("name", new ScalarValue("coffee"))
+                ])),
+                new LogEventProperty("domains", new StructureValue(
+                [
+                    new LogEventProperty("example.com", new ScalarValue(42)),
+                    new LogEventProperty("datalust.co", new ScalarValue(43))
+                ]))
+            ]),
+            ["scope"] = new StructureValue([]),
+            ["scope.name"] = new ScalarValue("Gerald"),
+            ["vegetable"] = new ScalarValue("Potato"),
+            ["Scope"] = new ScalarValue("Periscope"),
+            [".gitattributes"] = new ScalarValue("Text")
+        };
+
+        var actual = new UnflattenDottedPropertyNames().ProcessDottedPropertyNames(someDotted);
+
+        Assert.Equal(expected.Count, actual.Count);
+        foreach (var expectedProperty in expected)
+        {
+            Assert.True(actual.TryGetValue(expectedProperty.Key, out var actualProperty));
+            AssertEquivalentValue(expectedProperty.Value, expectedProperty.Value);
+        }
+    }
+
+    [Theory]
+    [InlineData("", false)]
+    [InlineData(".", false)]
+    [InlineData("..", false)]
+    [InlineData(".a", false)]
+    [InlineData("a.", false)]
+    [InlineData("a..b", false)]
+    [InlineData("a.b..c", false)]
+    [InlineData("a.b.", false)]
+    [InlineData("a. .b", false)]
+    [InlineData("1.0", false)]
+    [InlineData("1", false)]
+    [InlineData("a", false)]
+    [InlineData("abc", false)]
+    [InlineData("a.b", true)]
+    [InlineData("a1.bc._._xd.e_", true)]
+    public void OnlyProcessesValidDottedNames(string key, bool isValid)
+    {
+        var actual = UnflattenDottedPropertyNames.IsDottedIdentifier(key);
+        Assert.Equal(isValid, actual);
+    }
+
+    static void AssertEquivalentValue(LogEventPropertyValue expected, LogEventPropertyValue actual)
+    {
+        switch (expected, actual)
+        {
+            case (ScalarValue expectedScalar, ScalarValue actualScalar):
+            {
+                Assert.Equal(expectedScalar.Value, actualScalar.Value);
+                break;
+            }
+            case (StructureValue expectedStructure, StructureValue actualStructure):
+            {
+                Assert.Equal(expectedStructure.TypeTag, actualStructure.TypeTag);
+                Assert.Equal(expectedStructure.Properties.Count, actualStructure.Properties.Count);
+                var actualProperties = actualStructure.Properties.ToDictionary(p => p.Name, p => p.Value);
+                foreach (var expectedProperty in expectedStructure.Properties)
+                {
+                    var actualValue = Assert.Contains(expectedProperty.Name, actualProperties);
+                    AssertEquivalentValue(expectedProperty.Value, actualValue);
+                }
+                break;
+            }
+            default:
+            {
+                Assert.Equal(expected, actual);
+                break;
+            }
+        }
+    }
+}