From 39de304a5e7e34620c105aa0a18ebeb4ea9f21c9 Mon Sep 17 00:00:00 2001 From: Dion Date: Fri, 29 Dec 2023 10:34:45 +0100 Subject: [PATCH 01/20] WIP --- .../Models/AppSettings.cs | 4 ++ .../ApplicationInsightsExtension.cs | 2 +- .../Extensions/OpenTelemetryExtension.cs | 42 +++++++++++++++++++ .../starsky.foundation.webtelemetry.csproj | 5 +++ 4 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 starsky/starsky.foundation.webtelemetry/Extensions/OpenTelemetryExtension.cs diff --git a/starsky/starsky.foundation.platform/Models/AppSettings.cs b/starsky/starsky.foundation.platform/Models/AppSettings.cs index 4d15b3a4d9..2a2ce3e59a 100644 --- a/starsky/starsky.foundation.platform/Models/AppSettings.cs +++ b/starsky/starsky.foundation.platform/Models/AppSettings.cs @@ -879,6 +879,10 @@ private string AssemblyDirectoryReplacer(string value) /// public bool? ExiftoolSkipDownloadOnStartup { get; set; } = false; + public string OpenTelemetryEndpoint { get; set; } + + public string OpenTelemetryHeader { get; set; } + /// AppSettings duplicated /// /// Duplicate this item in memory. AND remove _databaseConnection diff --git a/starsky/starsky.foundation.webtelemetry/Extensions/ApplicationInsightsExtension.cs b/starsky/starsky.foundation.webtelemetry/Extensions/ApplicationInsightsExtension.cs index 5a1e2cd2ac..b0e6ec224e 100644 --- a/starsky/starsky.foundation.webtelemetry/Extensions/ApplicationInsightsExtension.cs +++ b/starsky/starsky.foundation.webtelemetry/Extensions/ApplicationInsightsExtension.cs @@ -20,7 +20,7 @@ public static class ApplicationInsightsExtension /// to use for ApplicationInsights InstrumentationKey public static void AddMonitoring(this IServiceCollection services, AppSettings appSettings) { - if ( string.IsNullOrWhiteSpace(appSettings.ApplicationInsightsConnectionString) ) + if ( string.IsNullOrWhiteSpace(appSettings.OpenTelemetryEndpoint) ) { return; } diff --git a/starsky/starsky.foundation.webtelemetry/Extensions/OpenTelemetryExtension.cs b/starsky/starsky.foundation.webtelemetry/Extensions/OpenTelemetryExtension.cs new file mode 100644 index 0000000000..c8aa225c4f --- /dev/null +++ b/starsky/starsky.foundation.webtelemetry/Extensions/OpenTelemetryExtension.cs @@ -0,0 +1,42 @@ +using System.Runtime.CompilerServices; +using Microsoft.Extensions.DependencyInjection; +using starsky.foundation.platform.Models; + +[assembly: InternalsVisibleTo("starskytest")] +namespace starsky.foundation.webtelemetry.Extensions; + +public static class OpenTelemetryExtension +{ + /// + /// Add Metrics & Monitoring for OpenTelemetry + /// + /// collection service + /// to use for ApplicationInsights InstrumentationKey + public static void AddOpenTelemetryMonitoring( + this IServiceCollection services, AppSettings appSettings) + { + if ( string.IsNullOrWhiteSpace(appSettings.ApplicationInsightsConnectionString) ) + { + return; + } + + const string serviceName = "roll-dice"; + + services.builder.Logging.AddOpenTelemetry(options => + { + options + .SetResourceBuilder( + ResourceBuilder.CreateDefault() + .AddService(serviceName)) + .AddConsoleExporter(); + }); + builder.Services.AddOpenTelemetry() + .ConfigureResource(resource => resource.AddService(serviceName)) + .WithTracing(tracing => tracing + .AddAspNetCoreInstrumentation() + .AddConsoleExporter()) + .WithMetrics(metrics => metrics + .AddAspNetCoreInstrumentation() + .AddConsoleExporter()); + } +} diff --git a/starsky/starsky.foundation.webtelemetry/starsky.foundation.webtelemetry.csproj b/starsky/starsky.foundation.webtelemetry/starsky.foundation.webtelemetry.csproj index 680b904238..a6459d670d 100644 --- a/starsky/starsky.foundation.webtelemetry/starsky.foundation.webtelemetry.csproj +++ b/starsky/starsky.foundation.webtelemetry/starsky.foundation.webtelemetry.csproj @@ -23,6 +23,11 @@ + + + + + From e6e3d37dd2716ff6482f65b608a9905b28a99376 Mon Sep 17 00:00:00 2001 From: Dion Date: Sun, 31 Dec 2023 16:27:59 +0100 Subject: [PATCH 02/20] Work in progress --- .../Extensions/OpenTelemetryExtension.cs | 65 +++++++++++++------ .../Helpers/SetupLogging.cs | 23 ++++++- .../starsky.foundation.webtelemetry.csproj | 6 +- .../default-init-launchSettings.json | 4 +- starsky/starsky/Startup.cs | 3 +- starsky/starskyadmincli/Program.cs | 2 +- starsky/starskydemoseedcli/Program.cs | 2 +- starsky/starskygeocli/Program.cs | 2 +- starsky/starskyimportercli/Program.cs | 2 +- starsky/starskysynchronizecli/Program.cs | 2 +- .../Helpers/SetupLoggingTest.cs | 2 +- starsky/starskythumbnailcli/Program.cs | 2 +- starsky/starskythumbnailmetacli/Program.cs | 2 +- 13 files changed, 83 insertions(+), 34 deletions(-) diff --git a/starsky/starsky.foundation.webtelemetry/Extensions/OpenTelemetryExtension.cs b/starsky/starsky.foundation.webtelemetry/Extensions/OpenTelemetryExtension.cs index c8aa225c4f..5fbfd5d684 100644 --- a/starsky/starsky.foundation.webtelemetry/Extensions/OpenTelemetryExtension.cs +++ b/starsky/starsky.foundation.webtelemetry/Extensions/OpenTelemetryExtension.cs @@ -1,5 +1,12 @@ +using System; +using System.Collections.Generic; +using System.Reflection; using System.Runtime.CompilerServices; using Microsoft.Extensions.DependencyInjection; +using OpenTelemetry.Exporter; +using OpenTelemetry.Metrics; +using OpenTelemetry.Resources; +using OpenTelemetry.Trace; using starsky.foundation.platform.Models; [assembly: InternalsVisibleTo("starskytest")] @@ -15,28 +22,48 @@ public static class OpenTelemetryExtension public static void AddOpenTelemetryMonitoring( this IServiceCollection services, AppSettings appSettings) { - if ( string.IsNullOrWhiteSpace(appSettings.ApplicationInsightsConnectionString) ) + if ( string.IsNullOrWhiteSpace(appSettings.OpenTelemetryEndpoint) ) { return; } - - const string serviceName = "roll-dice"; - services.builder.Logging.AddOpenTelemetry(options => - { - options - .SetResourceBuilder( - ResourceBuilder.CreateDefault() - .AddService(serviceName)) - .AddConsoleExporter(); - }); - builder.Services.AddOpenTelemetry() - .ConfigureResource(resource => resource.AddService(serviceName)) - .WithTracing(tracing => tracing - .AddAspNetCoreInstrumentation() - .AddConsoleExporter()) - .WithMetrics(metrics => metrics - .AddAspNetCoreInstrumentation() - .AddConsoleExporter()); + services.AddOpenTelemetry() + .ConfigureResource(resource => resource.AddService( + serviceNamespace: appSettings.Name, + serviceName: appSettings.Name, + serviceVersion: Assembly.GetEntryAssembly()?.GetName().Version + ?.ToString(), + serviceInstanceId: Environment.MachineName + ).AddAttributes(new Dictionary + { + { + "deployment.environment", + Environment.GetEnvironmentVariable("ASPNETCORE_ENVIRONMENT") + ! + } + })) + .WithTracing(tracing => tracing.AddAspNetCoreInstrumentation() + .AddConsoleExporter() + .AddOtlpExporter( + o => + { + o.Endpoint = new Uri(appSettings.OpenTelemetryEndpoint); + o.Protocol = OtlpExportProtocol.HttpProtobuf; + o.Headers = appSettings.OpenTelemetryHeader; + } + ) + ) + .WithMetrics(metrics => + metrics.AddAspNetCoreInstrumentation() + .AddRuntimeInstrumentation() + .AddConsoleExporter() + .AddOtlpExporter( + o => + { + o.Endpoint = new Uri(appSettings.OpenTelemetryEndpoint); + o.Protocol = OtlpExportProtocol.HttpProtobuf; + o.Headers = appSettings.OpenTelemetryHeader; + }) + ); } } diff --git a/starsky/starsky.foundation.webtelemetry/Helpers/SetupLogging.cs b/starsky/starsky.foundation.webtelemetry/Helpers/SetupLogging.cs index 5f4ac1961c..8314d4ed61 100644 --- a/starsky/starsky.foundation.webtelemetry/Helpers/SetupLogging.cs +++ b/starsky/starsky.foundation.webtelemetry/Helpers/SetupLogging.cs @@ -1,6 +1,9 @@ +using System; using System.Diagnostics.CodeAnalysis; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; +using OpenTelemetry.Exporter; +using OpenTelemetry.Logs; using starsky.foundation.platform.Interfaces; using starsky.foundation.platform.Models; using starsky.foundation.platform.Services; @@ -10,14 +13,26 @@ namespace starsky.foundation.webtelemetry.Helpers public static class SetupLogging { [SuppressMessage("Usage", "S4792:Make sure that this logger's configuration is safe.")] - public static void AddApplicationInsightsLogging(this IServiceCollection services, AppSettings appSettings) + public static void AddTelemetryLogging(this IServiceCollection services, AppSettings appSettings) { services.AddLogging(logging => { logging.ClearProviders(); logging.AddConsole(); - - // Skip when is Development + + if ( !string.IsNullOrEmpty(appSettings.OpenTelemetryEndpoint) ) + { + logging.AddOpenTelemetry(builder => builder.AddOtlpExporter( + "logging", + options => + { + options.Protocol = OtlpExportProtocol.HttpProtobuf; + options.Headers = appSettings.OpenTelemetryHeader; + options.Endpoint = new Uri(appSettings.OpenTelemetryEndpoint); + })); + } + + // Remove when ApplicationInsights is phased out if (appSettings.ApplicationInsightsLog != true || string.IsNullOrWhiteSpace(appSettings.ApplicationInsightsConnectionString)) return; @@ -27,6 +42,8 @@ public static void AddApplicationInsightsLogging(this IServiceCollection service telemetryConfiguration.ConnectionString = appSettings.ApplicationInsightsConnectionString; }, _ => { }); + // End Remove when ApplicationInsights is phased out + }); services.AddScoped(); diff --git a/starsky/starsky.foundation.webtelemetry/starsky.foundation.webtelemetry.csproj b/starsky/starsky.foundation.webtelemetry/starsky.foundation.webtelemetry.csproj index 56b9731306..6e9925fd7d 100644 --- a/starsky/starsky.foundation.webtelemetry/starsky.foundation.webtelemetry.csproj +++ b/starsky/starsky.foundation.webtelemetry/starsky.foundation.webtelemetry.csproj @@ -25,9 +25,11 @@ + - - + + + diff --git a/starsky/starsky/Properties/default-init-launchSettings.json b/starsky/starsky/Properties/default-init-launchSettings.json index 34c8eede26..89fe3580fc 100644 --- a/starsky/starsky/Properties/default-init-launchSettings.json +++ b/starsky/starsky/Properties/default-init-launchSettings.json @@ -27,7 +27,9 @@ "app__DemoUnsafeDeleteStorageFolder": "false", "app__useSystemTrash": "true", "app__accountRolesByEmailRegisterOverwrite__demo@qdraw.nl": "Administrator", - "app__ThumbnailGenerationIntervalInMinutes": "15" + "app__ThumbnailGenerationIntervalInMinutes": "15", + "_app__OpenTelemetryEndpoint": "http://localhost:9411/api/v2/spans", + "app__OpenTelemetryHeader": "api-key=test" } } } diff --git a/starsky/starsky/Startup.cs b/starsky/starsky/Startup.cs index dddc1a132b..adfa4e5ec9 100644 --- a/starsky/starsky/Startup.cs +++ b/starsky/starsky/Startup.cs @@ -71,9 +71,10 @@ public void ConfigureServices(IServiceCollection services) // Detect Application Insights (used in next SetupDatabaseTypes) services.AddMonitoring(_appSettings); + services.AddOpenTelemetryMonitoring(_appSettings); // LoggerFactory - services.AddApplicationInsightsLogging(_appSettings); + services.AddTelemetryLogging(_appSettings); var foundationDatabaseName = typeof(ApplicationDbContext).Assembly.FullName?.Split(",").FirstOrDefault(); new SetupDatabaseTypes(_appSettings,services).BuilderDb(foundationDatabaseName); diff --git a/starsky/starskyadmincli/Program.cs b/starsky/starskyadmincli/Program.cs index 102566069e..0a89fa8dd6 100644 --- a/starsky/starskyadmincli/Program.cs +++ b/starsky/starskyadmincli/Program.cs @@ -38,7 +38,7 @@ internal static async Task Main(string[] args) var appSettings = serviceProvider.GetRequiredService(); services.AddMonitoringWorkerService(appSettings, AppSettings.StarskyAppType.Geo); - services.AddApplicationInsightsLogging(appSettings); + services.AddTelemetryLogging(appSettings); var webLogger = serviceProvider.GetRequiredService(); diff --git a/starsky/starskydemoseedcli/Program.cs b/starsky/starskydemoseedcli/Program.cs index de894549b3..a3b7d8e923 100644 --- a/starsky/starskydemoseedcli/Program.cs +++ b/starsky/starskydemoseedcli/Program.cs @@ -33,7 +33,7 @@ public static async Task Main(string[] args) var appSettings = serviceProvider.GetRequiredService(); services.AddMonitoringWorkerService(appSettings, AppSettings.StarskyAppType.DemoSeed); - services.AddApplicationInsightsLogging(appSettings); + services.AddTelemetryLogging(appSettings); new SetupDatabaseTypes(appSettings,services).BuilderDb(); serviceProvider = services.BuildServiceProvider(); diff --git a/starsky/starskygeocli/Program.cs b/starsky/starskygeocli/Program.cs index 721360bc1a..e912ac59bc 100644 --- a/starsky/starskygeocli/Program.cs +++ b/starsky/starskygeocli/Program.cs @@ -33,7 +33,7 @@ public static async Task Main(string[] args) var appSettings = serviceProvider.GetRequiredService(); services.AddMonitoringWorkerService(appSettings, AppSettings.StarskyAppType.Geo); - services.AddApplicationInsightsLogging(appSettings); + services.AddTelemetryLogging(appSettings); new SetupDatabaseTypes(appSettings,services).BuilderDb(); serviceProvider = services.BuildServiceProvider(); diff --git a/starsky/starskyimportercli/Program.cs b/starsky/starskyimportercli/Program.cs index 419666c42f..d59fdcfe65 100644 --- a/starsky/starskyimportercli/Program.cs +++ b/starsky/starskyimportercli/Program.cs @@ -32,7 +32,7 @@ public static async Task Main(string[] args) var appSettings = serviceProvider.GetRequiredService(); services.AddMonitoringWorkerService(appSettings, AppSettings.StarskyAppType.Importer); - services.AddApplicationInsightsLogging(appSettings); + services.AddTelemetryLogging(appSettings); new SetupDatabaseTypes(appSettings,services).BuilderDb(); serviceProvider = services.BuildServiceProvider(); diff --git a/starsky/starskysynchronizecli/Program.cs b/starsky/starskysynchronizecli/Program.cs index 197b10e56a..413ba31bc8 100644 --- a/starsky/starskysynchronizecli/Program.cs +++ b/starsky/starskysynchronizecli/Program.cs @@ -32,7 +32,7 @@ public static async Task Main(string[] args) var appSettings = serviceProvider.GetRequiredService(); services.AddMonitoringWorkerService(appSettings, AppSettings.StarskyAppType.Sync); - services.AddApplicationInsightsLogging(appSettings); + services.AddTelemetryLogging(appSettings); new SetupDatabaseTypes(appSettings,services).BuilderDb(); serviceProvider = services.BuildServiceProvider(); diff --git a/starsky/starskytest/starsky.foundation.webtelemetry/Helpers/SetupLoggingTest.cs b/starsky/starskytest/starsky.foundation.webtelemetry/Helpers/SetupLoggingTest.cs index 225c6f5d56..91609ae8e5 100644 --- a/starsky/starskytest/starsky.foundation.webtelemetry/Helpers/SetupLoggingTest.cs +++ b/starsky/starskytest/starsky.foundation.webtelemetry/Helpers/SetupLoggingTest.cs @@ -16,7 +16,7 @@ public void AddApplicationInsightsLoggingTest() { var testGuid = Guid.NewGuid().ToString(); IServiceCollection services = new ServiceCollection(); - services.AddApplicationInsightsLogging(new AppSettings + services.AddTelemetryLogging(new AppSettings { ApplicationInsightsLog = true, ApplicationInsightsConnectionString = $"InstrumentationKey={testGuid}" diff --git a/starsky/starskythumbnailcli/Program.cs b/starsky/starskythumbnailcli/Program.cs index 58f3d93b82..76592278bb 100644 --- a/starsky/starskythumbnailcli/Program.cs +++ b/starsky/starskythumbnailcli/Program.cs @@ -31,7 +31,7 @@ public static async Task Main(string[] args) var appSettings = serviceProvider.GetRequiredService(); services.AddMonitoringWorkerService(appSettings, AppSettings.StarskyAppType.Thumbnail); - services.AddApplicationInsightsLogging(appSettings); + services.AddTelemetryLogging(appSettings); new SetupDatabaseTypes(appSettings,services).BuilderDb(); serviceProvider = services.BuildServiceProvider(); diff --git a/starsky/starskythumbnailmetacli/Program.cs b/starsky/starskythumbnailmetacli/Program.cs index 73ef8dd50e..59497480ed 100644 --- a/starsky/starskythumbnailmetacli/Program.cs +++ b/starsky/starskythumbnailmetacli/Program.cs @@ -32,7 +32,7 @@ public static async Task Main(string[] args) var appSettings = serviceProvider.GetRequiredService(); services.AddMonitoringWorkerService(appSettings, AppSettings.StarskyAppType.MetaThumbnail); - services.AddApplicationInsightsLogging(appSettings); + services.AddTelemetryLogging(appSettings); new SetupDatabaseTypes(appSettings,services).BuilderDb(); serviceProvider = services.BuildServiceProvider(); From 073f743a438fc158fcb69b25ffb83e44b5b6ae78 Mon Sep 17 00:00:00 2001 From: Dion Date: Tue, 2 Jan 2024 17:02:36 +0100 Subject: [PATCH 03/20] undo --- .../Extensions/ApplicationInsightsExtension.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/starsky/starsky.foundation.webtelemetry/Extensions/ApplicationInsightsExtension.cs b/starsky/starsky.foundation.webtelemetry/Extensions/ApplicationInsightsExtension.cs index b0e6ec224e..5a1e2cd2ac 100644 --- a/starsky/starsky.foundation.webtelemetry/Extensions/ApplicationInsightsExtension.cs +++ b/starsky/starsky.foundation.webtelemetry/Extensions/ApplicationInsightsExtension.cs @@ -20,7 +20,7 @@ public static class ApplicationInsightsExtension /// to use for ApplicationInsights InstrumentationKey public static void AddMonitoring(this IServiceCollection services, AppSettings appSettings) { - if ( string.IsNullOrWhiteSpace(appSettings.OpenTelemetryEndpoint) ) + if ( string.IsNullOrWhiteSpace(appSettings.ApplicationInsightsConnectionString) ) { return; } From 4a7ef921f8adc7f45a6ead03d394b5e830090d8f Mon Sep 17 00:00:00 2001 From: Dion Date: Wed, 3 Jan 2024 20:42:04 +0100 Subject: [PATCH 04/20] add compare --- .../Helpers/AppSettingsCompareHelper.cs | 31 ++++++++++ .../Models/AppSettings.cs | 5 +- .../Models/OpenTelemetrySettings.cs | 44 +++++++++++++ .../Extensions/OpenTelemetryExtension.cs | 61 ++++++++++++------- .../Helpers/SetupLogging.cs | 17 ++++-- .../starsky.foundation.webtelemetry.csproj | 1 + .../Helpers/AppSettingsCompareHelperTest.cs | 43 +++++++++++++ 7 files changed, 171 insertions(+), 31 deletions(-) create mode 100644 starsky/starsky.foundation.platform/Models/OpenTelemetrySettings.cs diff --git a/starsky/starsky.foundation.platform/Helpers/AppSettingsCompareHelper.cs b/starsky/starsky.foundation.platform/Helpers/AppSettingsCompareHelper.cs index 44e2ce3e5c..7dbab1ad2e 100644 --- a/starsky/starsky.foundation.platform/Helpers/AppSettingsCompareHelper.cs +++ b/starsky/starsky.foundation.platform/Helpers/AppSettingsCompareHelper.cs @@ -33,11 +33,42 @@ public static List Compare(AppSettings sourceIndexItem, object? updateOb CompareMultipleSingleItems(propertyB, propertyInfoFromA, sourceIndexItem, updateObject, differenceList); CompareMultipleListDictionary(propertyB, propertyInfoFromA, sourceIndexItem, updateObject, differenceList); + CompareMultipleObjects(propertyB, propertyInfoFromA, sourceIndexItem, updateObject, differenceList); } return differenceList; } + private static void CompareMultipleObjects(PropertyInfo propertyB, PropertyInfo propertyInfoFromA, AppSettings sourceIndexItem, object updateObject, List differenceList) + { + if (propertyInfoFromA.PropertyType == typeof(OpenTelemetrySettings) && propertyB.PropertyType == typeof(OpenTelemetrySettings)) + { + var oldBoolValue = (OpenTelemetrySettings?)propertyInfoFromA.GetValue(sourceIndexItem, null); + var newBoolValue = (OpenTelemetrySettings?)propertyB.GetValue(updateObject, null); + CompareOpenTelemetrySettingsObject(propertyB.Name, sourceIndexItem, oldBoolValue, newBoolValue, differenceList); + } + } + + private static void CompareOpenTelemetrySettingsObject(string propertyName, AppSettings? sourceIndexItem, + OpenTelemetrySettings? oldKeyValuePairStringStringValue, + OpenTelemetrySettings? newKeyValuePairStringStringValue, ICollection differenceList) + { + if ( oldKeyValuePairStringStringValue == null || + newKeyValuePairStringStringValue == null) + { + return; + } + + if ( oldKeyValuePairStringStringValue.Equals( + newKeyValuePairStringStringValue) ) + { + return; + } + + sourceIndexItem?.GetType().GetProperty(propertyName)?.SetValue(sourceIndexItem, newKeyValuePairStringStringValue, null); + differenceList.Add(propertyName.ToLowerInvariant()); + } + private static void CompareMultipleSingleItems(PropertyInfo propertyB, PropertyInfo propertyInfoFromA, AppSettings sourceIndexItem, object updateObject, diff --git a/starsky/starsky.foundation.platform/Models/AppSettings.cs b/starsky/starsky.foundation.platform/Models/AppSettings.cs index 2a2ce3e59a..9cd8bb3a44 100644 --- a/starsky/starsky.foundation.platform/Models/AppSettings.cs +++ b/starsky/starsky.foundation.platform/Models/AppSettings.cs @@ -879,9 +879,8 @@ private string AssemblyDirectoryReplacer(string value) /// public bool? ExiftoolSkipDownloadOnStartup { get; set; } = false; - public string OpenTelemetryEndpoint { get; set; } - - public string OpenTelemetryHeader { get; set; } + public OpenTelemetrySettings OpenTelemetry { get; set; } = + new OpenTelemetrySettings(); /// AppSettings duplicated /// diff --git a/starsky/starsky.foundation.platform/Models/OpenTelemetrySettings.cs b/starsky/starsky.foundation.platform/Models/OpenTelemetrySettings.cs new file mode 100644 index 0000000000..2fdbe8cf06 --- /dev/null +++ b/starsky/starsky.foundation.platform/Models/OpenTelemetrySettings.cs @@ -0,0 +1,44 @@ +namespace starsky.foundation.platform.Models; + +public class OpenTelemetrySettings +{ + public string Header { get; set; } + + public string ServiceName { get; set; } + public string TracesEndpoint { get; set; } + public string TracesHeader { get; set; } + + public string MetricsEndpoint { get; set; } + public string MetricsHeader { get; set; } + + public string LogsEndpoint { get; set; } + public string LogsHeader { get; set; } + + public string GetServiceName() + { + return string.IsNullOrWhiteSpace(ServiceName) + ? "Starsky" + : ServiceName ; + } + + public string GetLogsHeader() + { + return string.IsNullOrWhiteSpace(LogsHeader) + ? Header + : LogsHeader; + } + + public string GetMetricsHeader() + { + return string.IsNullOrWhiteSpace(MetricsHeader) + ? Header + : MetricsHeader; + } + + public string GetTracesHeader() + { + return string.IsNullOrWhiteSpace(TracesHeader) + ? Header + : TracesHeader; + } +} diff --git a/starsky/starsky.foundation.webtelemetry/Extensions/OpenTelemetryExtension.cs b/starsky/starsky.foundation.webtelemetry/Extensions/OpenTelemetryExtension.cs index 5fbfd5d684..d4b81b116c 100644 --- a/starsky/starsky.foundation.webtelemetry/Extensions/OpenTelemetryExtension.cs +++ b/starsky/starsky.foundation.webtelemetry/Extensions/OpenTelemetryExtension.cs @@ -7,6 +7,7 @@ using OpenTelemetry.Metrics; using OpenTelemetry.Resources; using OpenTelemetry.Trace; +using starsky.foundation.platform.Interfaces; using starsky.foundation.platform.Models; [assembly: InternalsVisibleTo("starskytest")] @@ -22,12 +23,7 @@ public static class OpenTelemetryExtension public static void AddOpenTelemetryMonitoring( this IServiceCollection services, AppSettings appSettings) { - if ( string.IsNullOrWhiteSpace(appSettings.OpenTelemetryEndpoint) ) - { - return; - } - - services.AddOpenTelemetry() + var telemetryBuilder = services.AddOpenTelemetry() .ConfigureResource(resource => resource.AddService( serviceNamespace: appSettings.Name, serviceName: appSettings.Name, @@ -38,32 +34,51 @@ public static void AddOpenTelemetryMonitoring( { { "deployment.environment", - Environment.GetEnvironmentVariable("ASPNETCORE_ENVIRONMENT") - ! + Environment.GetEnvironmentVariable("ASPNETCORE_ENVIRONMENT")! } - })) - .WithTracing(tracing => tracing.AddAspNetCoreInstrumentation() - .AddConsoleExporter() + })); + + if ( !string.IsNullOrWhiteSpace(appSettings.OpenTelemetry.TracesEndpoint) ) + { + telemetryBuilder.WithTracing(tracing => tracing + .AddAspNetCoreInstrumentation() .AddOtlpExporter( o => { - o.Endpoint = new Uri(appSettings.OpenTelemetryEndpoint); + o.Endpoint = + new Uri(appSettings.OpenTelemetry.TracesEndpoint); o.Protocol = OtlpExportProtocol.HttpProtobuf; - o.Headers = appSettings.OpenTelemetryHeader; + o.Headers = appSettings.OpenTelemetry.GetTracesHeader(); } + ).SetResourceBuilder( + ResourceBuilder.CreateDefault() + .AddService(appSettings.OpenTelemetry.GetServiceName()) ) - ) - .WithMetrics(metrics => - metrics.AddAspNetCoreInstrumentation() - .AddRuntimeInstrumentation() - .AddConsoleExporter() - .AddOtlpExporter( - o => + ); + } + + if ( string.IsNullOrWhiteSpace( + appSettings.OpenTelemetry.MetricsEndpoint) ) + { + return; + } + + telemetryBuilder.WithMetrics(metrics => + metrics.AddAspNetCoreInstrumentation() + .AddRuntimeInstrumentation() + .AddHttpClientInstrumentation() + .AddOtlpExporter( + o => { - o.Endpoint = new Uri(appSettings.OpenTelemetryEndpoint); + o.Endpoint = new Uri(appSettings.OpenTelemetry.MetricsEndpoint); o.Protocol = OtlpExportProtocol.HttpProtobuf; - o.Headers = appSettings.OpenTelemetryHeader; + o.Headers = appSettings.OpenTelemetry.GetMetricsHeader(); }) - ); + .SetResourceBuilder( + ResourceBuilder.CreateDefault() + .AddService(appSettings.OpenTelemetry.GetServiceName()) + ) + ); + } } diff --git a/starsky/starsky.foundation.webtelemetry/Helpers/SetupLogging.cs b/starsky/starsky.foundation.webtelemetry/Helpers/SetupLogging.cs index 8314d4ed61..040e8ce6aa 100644 --- a/starsky/starsky.foundation.webtelemetry/Helpers/SetupLogging.cs +++ b/starsky/starsky.foundation.webtelemetry/Helpers/SetupLogging.cs @@ -4,6 +4,7 @@ using Microsoft.Extensions.Logging; using OpenTelemetry.Exporter; using OpenTelemetry.Logs; +using OpenTelemetry.Resources; using starsky.foundation.platform.Interfaces; using starsky.foundation.platform.Models; using starsky.foundation.platform.Services; @@ -20,16 +21,22 @@ public static void AddTelemetryLogging(this IServiceCollection services, AppSett logging.ClearProviders(); logging.AddConsole(); - if ( !string.IsNullOrEmpty(appSettings.OpenTelemetryEndpoint) ) + if ( !string.IsNullOrEmpty(appSettings.OpenTelemetry.LogsEndpoint) ) { - logging.AddOpenTelemetry(builder => builder.AddOtlpExporter( + logging.AddOpenTelemetry(builder => + builder.AddOtlpExporter( "logging", options => { options.Protocol = OtlpExportProtocol.HttpProtobuf; - options.Headers = appSettings.OpenTelemetryHeader; - options.Endpoint = new Uri(appSettings.OpenTelemetryEndpoint); - })); + options.Headers = appSettings.OpenTelemetry.GetLogsHeader(); + options.Endpoint = new Uri(appSettings.OpenTelemetry.LogsEndpoint); + }) + .SetResourceBuilder( + ResourceBuilder.CreateDefault() + .AddService(appSettings.OpenTelemetry.GetServiceName()) + ) + ); } // Remove when ApplicationInsights is phased out diff --git a/starsky/starsky.foundation.webtelemetry/starsky.foundation.webtelemetry.csproj b/starsky/starsky.foundation.webtelemetry/starsky.foundation.webtelemetry.csproj index 6e9925fd7d..cc8776c41b 100644 --- a/starsky/starsky.foundation.webtelemetry/starsky.foundation.webtelemetry.csproj +++ b/starsky/starsky.foundation.webtelemetry/starsky.foundation.webtelemetry.csproj @@ -29,6 +29,7 @@ + diff --git a/starsky/starskytest/starsky.foundation.platform/Helpers/AppSettingsCompareHelperTest.cs b/starsky/starskytest/starsky.foundation.platform/Helpers/AppSettingsCompareHelperTest.cs index df49b04cb4..6effbb8f0e 100644 --- a/starsky/starskytest/starsky.foundation.platform/Helpers/AppSettingsCompareHelperTest.cs +++ b/starsky/starskytest/starsky.foundation.platform/Helpers/AppSettingsCompareHelperTest.cs @@ -477,5 +477,48 @@ public void CompareInt_NotFound() 2, list); Assert.IsNotNull(list); } + + [TestMethod] + public void OpenTelemetrySettings() + { + var source = new AppSettings + { + OpenTelemetry = new OpenTelemetrySettings + { + Header = "source/test", + TracesEndpoint = "source/traces", + TracesHeader = "source/traces", + MetricsEndpoint = "source/metrics", + MetricsHeader = "source/metrics", + LogsEndpoint = "source/logs", + LogsHeader = "source/logs" + } + }; + + var to = new AppSettings + { + OpenTelemetry = new OpenTelemetrySettings + { + Header = "to/test", + TracesEndpoint = "to/traces", + TracesHeader = "to/traces", + MetricsEndpoint = "to/metrics", + MetricsHeader = "to/metrics", + LogsEndpoint = "to/logs", + LogsHeader = "to/logs" + } + }; + + AppSettingsCompareHelper.Compare(source, to); + + Assert.AreEqual(source.OpenTelemetry.Header, to.OpenTelemetry.Header); + Assert.AreEqual(source.OpenTelemetry.TracesEndpoint, to.OpenTelemetry.TracesEndpoint); + Assert.AreEqual(source.OpenTelemetry.TracesHeader, to.OpenTelemetry.TracesHeader); + Assert.AreEqual(source.OpenTelemetry.MetricsEndpoint, to.OpenTelemetry.MetricsEndpoint); + Assert.AreEqual(source.OpenTelemetry.MetricsHeader, to.OpenTelemetry.MetricsHeader); + Assert.AreEqual(source.OpenTelemetry.LogsEndpoint, to.OpenTelemetry.LogsEndpoint); + Assert.AreEqual(source.OpenTelemetry.LogsHeader, to.OpenTelemetry.LogsHeader); + + } } } From 9e39a44a8391bba10d0a427feee0d854e5588e26 Mon Sep 17 00:00:00 2001 From: Dion Date: Thu, 4 Jan 2024 17:15:54 +0100 Subject: [PATCH 05/20] add tests --- .../Helpers/AppSettingsCompareHelper.cs | 10 +- .../Models/AppSettings.cs | 34 ++++++ .../Extensions/OpenTelemetryExtension.cs | 20 ++-- .../Helpers/OpenTelemetryExtensionTest.cs | 106 ++++++++++++++++++ 4 files changed, 156 insertions(+), 14 deletions(-) create mode 100644 starsky/starskytest/starsky.foundation.webtelemetry/Helpers/OpenTelemetryExtensionTest.cs diff --git a/starsky/starsky.foundation.platform/Helpers/AppSettingsCompareHelper.cs b/starsky/starsky.foundation.platform/Helpers/AppSettingsCompareHelper.cs index 7dbab1ad2e..0b4180bb3b 100644 --- a/starsky/starsky.foundation.platform/Helpers/AppSettingsCompareHelper.cs +++ b/starsky/starsky.foundation.platform/Helpers/AppSettingsCompareHelper.cs @@ -54,13 +54,9 @@ private static void CompareOpenTelemetrySettingsObject(string propertyName, AppS OpenTelemetrySettings? newKeyValuePairStringStringValue, ICollection differenceList) { if ( oldKeyValuePairStringStringValue == null || - newKeyValuePairStringStringValue == null) - { - return; - } - - if ( oldKeyValuePairStringStringValue.Equals( - newKeyValuePairStringStringValue) ) + newKeyValuePairStringStringValue == null || + JsonSerializer.Serialize(oldKeyValuePairStringStringValue) == + JsonSerializer.Serialize(newKeyValuePairStringStringValue)) { return; } diff --git a/starsky/starsky.foundation.platform/Models/AppSettings.cs b/starsky/starsky.foundation.platform/Models/AppSettings.cs index 9cd8bb3a44..73946ff904 100644 --- a/starsky/starsky.foundation.platform/Models/AppSettings.cs +++ b/starsky/starsky.foundation.platform/Models/AppSettings.cs @@ -930,9 +930,43 @@ public AppSettings CloneToDisplay() } } + ReplaceOpenTelemetryData(appSettings); + return appSettings; } + private static void ReplaceOpenTelemetryData(AppSettings appSettings) + { + if ( appSettings.OpenTelemetry == null ) + { + return; + } + + if (!string.IsNullOrEmpty(appSettings.OpenTelemetry.Header) ) + { + appSettings.OpenTelemetry.Header = + CloneToDisplaySecurityWarning; + } + + if (!string.IsNullOrEmpty(appSettings.OpenTelemetry.MetricsHeader) ) + { + appSettings.OpenTelemetry.MetricsHeader = + CloneToDisplaySecurityWarning; + } + + if (!string.IsNullOrEmpty(appSettings.OpenTelemetry.LogsHeader) ) + { + appSettings.OpenTelemetry.LogsHeader = + CloneToDisplaySecurityWarning; + } + + if (!string.IsNullOrEmpty(appSettings.OpenTelemetry.TracesHeader) ) + { + appSettings.OpenTelemetry.TracesHeader = + CloneToDisplaySecurityWarning; + } + } + private static void ReplaceAppSettingsPublishProfilesCloneToDisplay(AppSettingsPublishProfiles value) { if ( !string.IsNullOrEmpty(value.Path) && value.Path != AppSettingsPublishProfiles.GetDefaultPath() ) diff --git a/starsky/starsky.foundation.webtelemetry/Extensions/OpenTelemetryExtension.cs b/starsky/starsky.foundation.webtelemetry/Extensions/OpenTelemetryExtension.cs index d4b81b116c..852134ae1d 100644 --- a/starsky/starsky.foundation.webtelemetry/Extensions/OpenTelemetryExtension.cs +++ b/starsky/starsky.foundation.webtelemetry/Extensions/OpenTelemetryExtension.cs @@ -7,7 +7,6 @@ using OpenTelemetry.Metrics; using OpenTelemetry.Resources; using OpenTelemetry.Trace; -using starsky.foundation.platform.Interfaces; using starsky.foundation.platform.Models; [assembly: InternalsVisibleTo("starskytest")] @@ -19,14 +18,14 @@ public static class OpenTelemetryExtension /// Add Metrics & Monitoring for OpenTelemetry /// /// collection service - /// to use for ApplicationInsights InstrumentationKey + /// to use for OpenTelemetry keys and info public static void AddOpenTelemetryMonitoring( this IServiceCollection services, AppSettings appSettings) { var telemetryBuilder = services.AddOpenTelemetry() .ConfigureResource(resource => resource.AddService( - serviceNamespace: appSettings.Name, - serviceName: appSettings.Name, + serviceNamespace: appSettings.OpenTelemetry.GetServiceName(), + serviceName: appSettings.OpenTelemetry.GetServiceName(), serviceVersion: Assembly.GetEntryAssembly()?.GetName().Version ?.ToString(), serviceInstanceId: Environment.MachineName @@ -34,14 +33,22 @@ public static void AddOpenTelemetryMonitoring( { { "deployment.environment", - Environment.GetEnvironmentVariable("ASPNETCORE_ENVIRONMENT")! + Environment.GetEnvironmentVariable("ASPNETCORE_ENVIRONMENT") ?? string.Empty } })); if ( !string.IsNullOrWhiteSpace(appSettings.OpenTelemetry.TracesEndpoint) ) { telemetryBuilder.WithTracing(tracing => tracing - .AddAspNetCoreInstrumentation() + .AddAspNetCoreInstrumentation(o => o.Filter = context => + { + if ( context.Request.Path.Value?.EndsWith("/realtime") == true && + context.Request.Path.Value?.EndsWith("/api/health") == true) + { + return false; + } + return true; + }) .AddOtlpExporter( o => { @@ -79,6 +86,5 @@ public static void AddOpenTelemetryMonitoring( .AddService(appSettings.OpenTelemetry.GetServiceName()) ) ); - } } diff --git a/starsky/starskytest/starsky.foundation.webtelemetry/Helpers/OpenTelemetryExtensionTest.cs b/starsky/starskytest/starsky.foundation.webtelemetry/Helpers/OpenTelemetryExtensionTest.cs new file mode 100644 index 0000000000..75cdcef4a1 --- /dev/null +++ b/starsky/starskytest/starsky.foundation.webtelemetry/Helpers/OpenTelemetryExtensionTest.cs @@ -0,0 +1,106 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using OpenTelemetry.Metrics; +using OpenTelemetry.Trace; +using starsky.foundation.platform.Models; +using starsky.foundation.webtelemetry.Extensions; + +namespace starskytest.starsky.foundation.webtelemetry.Helpers; + +[TestClass] +public class OpenTelemetryExtensionTest +{ + + [TestMethod] + public void ConfiguresTelemetryBuilder() + { + // Arrange + var services = new ServiceCollection(); + var appSettings = new AppSettings + { + OpenTelemetry = new OpenTelemetrySettings + { + // Set OpenTelemetry settings as needed for the test + TracesEndpoint = "https://example.com/traces", + MetricsEndpoint = "https://example.com/metrics" + } + }; + + // Act + services.AddOpenTelemetryMonitoring(appSettings); + + // Assert + var serviceProvider = services.BuildServiceProvider(); + + + // Verify tracing configuration + var tracerProvider = serviceProvider.GetRequiredService(); + Assert.IsNotNull(tracerProvider); + + // Verify metrics configuration + var meterProvider = serviceProvider.GetRequiredService(); + Assert.IsNotNull(meterProvider); + } + + [TestMethod] + public void ConfiguresTelemetryBuilder_Skip_WhenTraces() + { + // Arrange + var services = new ServiceCollection(); + var appSettings = new AppSettings + { + OpenTelemetry = new OpenTelemetrySettings + { + // Set OpenTelemetry settings as needed for the test + TracesEndpoint = null, + MetricsEndpoint = "https://example.com/metrics" + } + }; + + // Act + services.AddOpenTelemetryMonitoring(appSettings); + + // Assert + var serviceProvider = services.BuildServiceProvider(); + + + // Verify tracing configuration + var tracerProvider = serviceProvider.GetService(); + Assert.IsNull(tracerProvider); + + // Verify metrics configuration + var meterProvider = serviceProvider.GetRequiredService(); + Assert.IsNotNull(meterProvider); + } + + [TestMethod] + public void ConfiguresTelemetryBuilder_Skip_WhenMetrics() + { + // Arrange + var services = new ServiceCollection(); + var appSettings = new AppSettings + { + OpenTelemetry = new OpenTelemetrySettings + { + // Set OpenTelemetry settings as needed for the test + TracesEndpoint = "https://example.com/traces", + MetricsEndpoint = null + } + }; + + // Act + services.AddOpenTelemetryMonitoring(appSettings); + + // Assert + var serviceProvider = services.BuildServiceProvider(); + + + // Verify tracing configuration + var tracerProvider = serviceProvider.GetService(); + Assert.IsNotNull(tracerProvider); + + // Verify metrics configuration + var meterProvider = serviceProvider.GetService(); + Assert.IsNull(meterProvider); + } +} From 35312b51c63ea0dd798a3a1e06396edb158dd611 Mon Sep 17 00:00:00 2001 From: Dion Date: Thu, 4 Jan 2024 17:19:38 +0100 Subject: [PATCH 06/20] add test --- .../Helpers/SetupLoggingTest.cs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/starsky/starskytest/starsky.foundation.webtelemetry/Helpers/SetupLoggingTest.cs b/starsky/starskytest/starsky.foundation.webtelemetry/Helpers/SetupLoggingTest.cs index 91609ae8e5..d9bc576461 100644 --- a/starsky/starskytest/starsky.foundation.webtelemetry/Helpers/SetupLoggingTest.cs +++ b/starsky/starskytest/starsky.foundation.webtelemetry/Helpers/SetupLoggingTest.cs @@ -3,6 +3,7 @@ using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.ApplicationInsights; using Microsoft.VisualStudio.TestTools.UnitTesting; +using OpenTelemetry.Logs; using starsky.foundation.platform.Models; using starsky.foundation.webtelemetry.Helpers; @@ -27,4 +28,22 @@ public void AddApplicationInsightsLoggingTest() var type = build.GetRequiredService(); Assert.AreEqual(typeof(ApplicationInsightsLoggerProvider),type.GetType()); } + + [TestMethod] + public void OpenTelemetry() + { + IServiceCollection services = new ServiceCollection(); + services.AddTelemetryLogging(new AppSettings + { + OpenTelemetry = new OpenTelemetrySettings + { + LogsEndpoint = "https://test.me/v1/logs" + } + }); + + var build = services.BuildServiceProvider(); + + var type = build.GetRequiredService(); + Assert.AreEqual(typeof(OpenTelemetryLoggerProvider),type.GetType()); + } } From 9c04a0c42ccfa8725ae78694024d4d86ef48a703 Mon Sep 17 00:00:00 2001 From: Dion Date: Thu, 4 Jan 2024 17:24:44 +0100 Subject: [PATCH 07/20] update values --- .../Properties/default-init-launchSettings.json | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/starsky/starsky/Properties/default-init-launchSettings.json b/starsky/starsky/Properties/default-init-launchSettings.json index 89fe3580fc..c31f279aa8 100644 --- a/starsky/starsky/Properties/default-init-launchSettings.json +++ b/starsky/starsky/Properties/default-init-launchSettings.json @@ -15,10 +15,10 @@ "app__NoAccountLocalhost": "false", "app__IsAccountRegisterOpen": "true", "app__SyncOnStartup": "false", - "_app__storageFolder": "remove prefix underscore", - "_app__dependenciesFolder": "remove prefix underscore", - "_app__ApplicationInsightsConnectionString": "remove prefix underscore", - "_app__DatabaseConnection": "Server=localhost;port=12838;database=starsky;uid=starsky;pwd=ad12dc47-a411-4dce-92b3-2649a755426c;maximumpoolsize=30;", + "___app__storageFolder": "remove prefix underscores ___ and update value", + "___app__dependenciesFolder": "remove prefix underscores ___and update value", + "___app__ApplicationInsightsConnectionString": "remove prefix underscores ___ and update value", + "___app__DatabaseConnection": "Server=localhost;port=12838;database=starsky;uid=starsky;pwd=ad12dc47-a411-4dce-92b3-2649a755426c;maximumpoolsize=30;", "app__DatabaseType": "Sqlite", "app__ApplicationInsightsDatabaseTracking": "true", "app__ApplicationInsightsLog": "true", @@ -28,8 +28,11 @@ "app__useSystemTrash": "true", "app__accountRolesByEmailRegisterOverwrite__demo@qdraw.nl": "Administrator", "app__ThumbnailGenerationIntervalInMinutes": "15", - "_app__OpenTelemetryEndpoint": "http://localhost:9411/api/v2/spans", - "app__OpenTelemetryHeader": "api-key=test" + "___app__OpenTelemetry__TracesEndpoint": "http://localhost:4318/v1/traces", + "___app__OpenTelemetry__MetricsEndpoint": "http://localhost:4318/v1/metrics", + "___app__OpenTelemetry__LogsEndpoint": "http://localhost:4318/v1/logs", + "___app__OpenTelemetry__Header": "api-key=test", + "___app__OpenTelemetry__ServiceName": "starsky-dev" } } } From d446cfb60f6a7435f22fac595349768bbe12374d Mon Sep 17 00:00:00 2001 From: Dion Date: Thu, 4 Jan 2024 19:28:11 +0100 Subject: [PATCH 08/20] add tests --- .../Models/AppSettingsTest.cs | 47 +++++++++++++++++-- 1 file changed, 44 insertions(+), 3 deletions(-) diff --git a/starsky/starskytest/starsky.foundation.platform/Models/AppSettingsTest.cs b/starsky/starskytest/starsky.foundation.platform/Models/AppSettingsTest.cs index fd9ea35818..c2c5374b9e 100644 --- a/starsky/starskytest/starsky.foundation.platform/Models/AppSettingsTest.cs +++ b/starsky/starskytest/starsky.foundation.platform/Models/AppSettingsTest.cs @@ -1,11 +1,9 @@ using System; using System.Collections.Generic; using System.IO; -using System.Runtime.InteropServices; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.VisualStudio.TestTools.UnitTesting; -using Newtonsoft.Json.Linq; using starsky.foundation.platform.Extensions; using starsky.foundation.platform.Models; using starskytest.FakeCreateAn; @@ -363,7 +361,50 @@ public void AppSettings_CloneToDisplay_hideSecurityItems() Assert.AreEqual(AppSettings.CloneToDisplaySecurityWarning, display.WebFtp); Assert.AreEqual(AppSettings.CloneToDisplaySecurityWarning, display.ApplicationInsightsConnectionString); } + + [TestMethod] + public void AppSettings_CloneToDisplay_hideSecurityItems2() + { + var appSettings = new AppSettings + { + OpenTelemetry = new OpenTelemetrySettings + { + Header = "test", + LogsHeader = "test", + MetricsHeader = "test", + TracesHeader = "test" + } + }; + var display = appSettings.CloneToDisplay(); + + Assert.AreEqual(AppSettings.CloneToDisplaySecurityWarning, display.OpenTelemetry.Header); + Assert.AreEqual(AppSettings.CloneToDisplaySecurityWarning, display.OpenTelemetry.LogsHeader); + Assert.AreEqual(AppSettings.CloneToDisplaySecurityWarning, display.OpenTelemetry.MetricsHeader); + Assert.AreEqual(AppSettings.CloneToDisplaySecurityWarning, display.OpenTelemetry.TracesHeader); + } + [TestMethod] + public void AppSettings_CloneToDisplay_skipSecurityItems2() + { + var appSettings = new AppSettings + { + OpenTelemetry = new OpenTelemetrySettings + { + Header = null, + LogsHeader = null, + MetricsHeader = null, + TracesHeader = null, + } + }; + + var display = appSettings.CloneToDisplay(); + + Assert.IsNull(display.OpenTelemetry.Header); + Assert.IsNull(display.OpenTelemetry.LogsHeader); + Assert.IsNull(display.OpenTelemetry.MetricsHeader); + Assert.IsNull(display.OpenTelemetry.TracesHeader); + } + [TestMethod] public void AppSettings_CloneToDisplay_hideSecurityItems_PublishProfiles() { @@ -406,7 +447,7 @@ public void AppSettings_IsReadOnly_NullNoItemTest() public void PublishProfiles_Null() { var appSettings = new AppSettings {PublishProfiles = null}; - Assert.AreEqual(0, appSettings.PublishProfiles.Count ); + Assert.AreEqual(0, appSettings.PublishProfiles?.Count); } [TestMethod] From 99efb08be006baa5fe7c94ba7b84e1dbdc709ae2 Mon Sep 17 00:00:00 2001 From: Dion Date: Thu, 4 Jan 2024 19:32:45 +0100 Subject: [PATCH 09/20] add tests --- .../Models/AppSettingsTest.cs | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/starsky/starskytest/starsky.foundation.platform/Models/AppSettingsTest.cs b/starsky/starskytest/starsky.foundation.platform/Models/AppSettingsTest.cs index c2c5374b9e..0657ead96e 100644 --- a/starsky/starskytest/starsky.foundation.platform/Models/AppSettingsTest.cs +++ b/starsky/starskytest/starsky.foundation.platform/Models/AppSettingsTest.cs @@ -528,5 +528,26 @@ public void AccountRolesByEmailRegisterOverwrite_ValidRole2() Assert.AreEqual(2,appSettings.AccountRolesByEmailRegisterOverwrite.Count); Assert.AreEqual("Administrator", appSettings.AccountRolesByEmailRegisterOverwrite["bogusEmail2"]); } + + + [TestMethod] + public void DatabasePathToFilePath_Null() + { + var appSettings = new AppSettings(); + + var result = appSettings.DatabasePathToFilePath("\\test"); + + Assert.IsNull(result); + } + + [TestMethod] + public void DatabasePathToFilePath_NoNull() + { + var appSettings = new AppSettings(); + + var result = appSettings.DatabasePathToFilePath("\\test",false); + + Assert.IsNotNull(result); + } } } From faaaac7e59704b607c2b9be9c64494e492b1468d Mon Sep 17 00:00:00 2001 From: Dion Date: Thu, 4 Jan 2024 20:01:12 +0100 Subject: [PATCH 10/20] move filter outside main --- .../Extensions/OpenTelemetryExtension.cs | 21 ++++++++++-------- .../Helpers/OpenTelemetryExtensionTest.cs | 22 +++++++++++++++++++ 2 files changed, 34 insertions(+), 9 deletions(-) diff --git a/starsky/starsky.foundation.webtelemetry/Extensions/OpenTelemetryExtension.cs b/starsky/starsky.foundation.webtelemetry/Extensions/OpenTelemetryExtension.cs index 852134ae1d..9a4eceb468 100644 --- a/starsky/starsky.foundation.webtelemetry/Extensions/OpenTelemetryExtension.cs +++ b/starsky/starsky.foundation.webtelemetry/Extensions/OpenTelemetryExtension.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Reflection; using System.Runtime.CompilerServices; +using Microsoft.AspNetCore.Http; using Microsoft.Extensions.DependencyInjection; using OpenTelemetry.Exporter; using OpenTelemetry.Metrics; @@ -40,15 +41,7 @@ public static void AddOpenTelemetryMonitoring( if ( !string.IsNullOrWhiteSpace(appSettings.OpenTelemetry.TracesEndpoint) ) { telemetryBuilder.WithTracing(tracing => tracing - .AddAspNetCoreInstrumentation(o => o.Filter = context => - { - if ( context.Request.Path.Value?.EndsWith("/realtime") == true && - context.Request.Path.Value?.EndsWith("/api/health") == true) - { - return false; - } - return true; - }) + .AddAspNetCoreInstrumentation(o => o.Filter = FilterPath) .AddOtlpExporter( o => { @@ -87,4 +80,14 @@ public static void AddOpenTelemetryMonitoring( ) ); } + + internal static bool FilterPath(HttpContext context) + { + if ( context.Request.Path.Value?.EndsWith("/realtime") == true || + context.Request.Path.Value?.EndsWith("/api/health") == true) + { + return false; + } + return true; + } } diff --git a/starsky/starskytest/starsky.foundation.webtelemetry/Helpers/OpenTelemetryExtensionTest.cs b/starsky/starskytest/starsky.foundation.webtelemetry/Helpers/OpenTelemetryExtensionTest.cs index 75cdcef4a1..ec0e948741 100644 --- a/starsky/starskytest/starsky.foundation.webtelemetry/Helpers/OpenTelemetryExtensionTest.cs +++ b/starsky/starskytest/starsky.foundation.webtelemetry/Helpers/OpenTelemetryExtensionTest.cs @@ -1,3 +1,4 @@ +using Microsoft.AspNetCore.Http; using Microsoft.Extensions.DependencyInjection; using Microsoft.VisualStudio.TestTools.UnitTesting; using OpenTelemetry.Metrics; @@ -103,4 +104,25 @@ public void ConfiguresTelemetryBuilder_Skip_WhenMetrics() var meterProvider = serviceProvider.GetService(); Assert.IsNull(meterProvider); } + + [TestMethod] + public void FilterPathTestIsTrue() + { + var context = new DefaultHttpContext { Request = { Path = "/test" } }; + Assert.IsTrue(OpenTelemetryExtension.FilterPath(context)); + } + + [TestMethod] + public void FilterPathTestSkipHealth() + { + var context = new DefaultHttpContext { Request = { Path = "/api/health" } }; + Assert.IsFalse(OpenTelemetryExtension.FilterPath(context)); + } + + [TestMethod] + public void FilterPathTestSkipRealtime() + { + var context = new DefaultHttpContext { Request = { Path = "/realtime" } }; + Assert.IsFalse(OpenTelemetryExtension.FilterPath(context)); + } } From 26bed7b0c9bfca2528d7c5953621f363981a828f Mon Sep 17 00:00:00 2001 From: Dion Date: Thu, 4 Jan 2024 20:06:12 +0100 Subject: [PATCH 11/20] add test --- .../Models/OpenTelemetrySettingsTest.cs | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 starsky/starskytest/starsky.foundation.platform/Models/OpenTelemetrySettingsTest.cs diff --git a/starsky/starskytest/starsky.foundation.platform/Models/OpenTelemetrySettingsTest.cs b/starsky/starskytest/starsky.foundation.platform/Models/OpenTelemetrySettingsTest.cs new file mode 100644 index 0000000000..9db29dfe3e --- /dev/null +++ b/starsky/starskytest/starsky.foundation.platform/Models/OpenTelemetrySettingsTest.cs @@ -0,0 +1,57 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; +using starsky.foundation.platform.Models; + +namespace starskytest.starsky.foundation.platform.Models; + +[TestClass] +public class OpenTelemetrySettingsTest +{ + [TestMethod] + public void ServiceNameDefault() + { + var result = new OpenTelemetrySettings().GetServiceName(); + Assert.AreEqual("Starsky",result); + } + + [TestMethod] + public void GetLogsHeaderDefault() + { + var result = new OpenTelemetrySettings().GetLogsHeader(); + Assert.AreEqual(null,result); + } + + [TestMethod] + public void GetLogsHeaderFallback() + { + var result = new OpenTelemetrySettings{Header = "1"}.GetLogsHeader(); + Assert.AreEqual("1",result); + } + + [TestMethod] + public void GetMetricsHeaderDefault() + { + var result = new OpenTelemetrySettings().GetMetricsHeader(); + Assert.AreEqual(null,result); + } + + [TestMethod] + public void GetMetricsHeaderFallback() + { + var result = new OpenTelemetrySettings{Header = "1"}.GetMetricsHeader(); + Assert.AreEqual("1",result); + } + + [TestMethod] + public void GetTracesHeaderDefault() + { + var result = new OpenTelemetrySettings().GetTracesHeader(); + Assert.AreEqual(null,result); + } + + [TestMethod] + public void GetTracesHeaderFallback() + { + var result = new OpenTelemetrySettings{Header = "1"}.GetTracesHeader(); + Assert.AreEqual("1",result); + } +} From 4a23a2c8f4a49b5659c33f2eb1ab9ca8b30eddb3 Mon Sep 17 00:00:00 2001 From: Dion Date: Fri, 5 Jan 2024 20:25:59 +0100 Subject: [PATCH 12/20] some backend changes --- .../Extensions/OpenTelemetryExtension.cs | 14 ++++++++++++-- .../Services/ApplicationInsightsJsHelper.cs | 3 +++ .../Services/FilterStatusCodesInitializer.cs | 4 ++++ .../Services/TelemetryService.cs | 3 +++ starsky/starsky/clientapp/src/get-cookie.ts | 6 ++++++ starsky/starsky/clientapp/src/shared/fetch-post.ts | 11 ++--------- 6 files changed, 30 insertions(+), 11 deletions(-) create mode 100644 starsky/starsky/clientapp/src/get-cookie.ts diff --git a/starsky/starsky.foundation.webtelemetry/Extensions/OpenTelemetryExtension.cs b/starsky/starsky.foundation.webtelemetry/Extensions/OpenTelemetryExtension.cs index 9a4eceb468..3fb7a94de9 100644 --- a/starsky/starsky.foundation.webtelemetry/Extensions/OpenTelemetryExtension.cs +++ b/starsky/starsky.foundation.webtelemetry/Extensions/OpenTelemetryExtension.cs @@ -83,11 +83,21 @@ public static void AddOpenTelemetryMonitoring( internal static bool FilterPath(HttpContext context) { - if ( context.Request.Path.Value?.EndsWith("/realtime") == true || - context.Request.Path.Value?.EndsWith("/api/health") == true) + if ( (context.Request.Path.Value?.EndsWith("/realtime") == true || + context.Request.Path.Value?.EndsWith("/api/health") == true || + context.Request.Path.Value?.EndsWith("/api/health/details") == true || + context.Request.Path.Value?.EndsWith("/api/open-telemetry/trace") == true) + && context.Response.StatusCode == 200) { return false; } + + if ( context.Request.Path.Value?.EndsWith("/api/index") == true + && context.Response.StatusCode == 401) + { + return false; + } + return true; } } diff --git a/starsky/starsky.foundation.webtelemetry/Services/ApplicationInsightsJsHelper.cs b/starsky/starsky.foundation.webtelemetry/Services/ApplicationInsightsJsHelper.cs index 7fcd438f51..0f953370ab 100644 --- a/starsky/starsky.foundation.webtelemetry/Services/ApplicationInsightsJsHelper.cs +++ b/starsky/starsky.foundation.webtelemetry/Services/ApplicationInsightsJsHelper.cs @@ -8,6 +8,9 @@ namespace starsky.foundation.webtelemetry.Services { + /// + /// Remove when App insights is phased out + /// [Service(InjectionLifetime = InjectionLifetime.Scoped)] public sealed class ApplicationInsightsJsHelper { diff --git a/starsky/starsky.foundation.webtelemetry/Services/FilterStatusCodesInitializer.cs b/starsky/starsky.foundation.webtelemetry/Services/FilterStatusCodesInitializer.cs index e855300027..3bfedecbf8 100644 --- a/starsky/starsky.foundation.webtelemetry/Services/FilterStatusCodesInitializer.cs +++ b/starsky/starsky.foundation.webtelemetry/Services/FilterStatusCodesInitializer.cs @@ -5,6 +5,10 @@ namespace starsky.foundation.webtelemetry.Services { + /// + /// App Insights Filter Status Codes Initializer + /// Remove when App insights is phased out + /// [Service(typeof(ITelemetryInitializer), InjectionLifetime = InjectionLifetime.Singleton)] public sealed class FilterStatusCodesInitializer : ITelemetryInitializer { diff --git a/starsky/starsky.foundation.webtelemetry/Services/TelemetryService.cs b/starsky/starsky.foundation.webtelemetry/Services/TelemetryService.cs index ac49b129af..2531eb2d4d 100644 --- a/starsky/starsky.foundation.webtelemetry/Services/TelemetryService.cs +++ b/starsky/starsky.foundation.webtelemetry/Services/TelemetryService.cs @@ -7,6 +7,9 @@ namespace starsky.foundation.webtelemetry.Services { + /// + /// Remove when App insights is phased out + /// [Service(typeof(ITelemetryService), InjectionLifetime = InjectionLifetime.Singleton)] public sealed class TelemetryService : ITelemetryService { diff --git a/starsky/starsky/clientapp/src/get-cookie.ts b/starsky/starsky/clientapp/src/get-cookie.ts new file mode 100644 index 0000000000..e11015caf0 --- /dev/null +++ b/starsky/starsky/clientapp/src/get-cookie.ts @@ -0,0 +1,6 @@ +export function GetCookie(name: string): string { + const regex = new RegExp("(^| )" + name + "=([^;]+)"); + const match = regex.exec(document.cookie); + if (match) return match[2]; + return "X-XSRF-TOKEN"; +} diff --git a/starsky/starsky/clientapp/src/shared/fetch-post.ts b/starsky/starsky/clientapp/src/shared/fetch-post.ts index 2bf1d4ff4c..e1fd82c0fc 100644 --- a/starsky/starsky/clientapp/src/shared/fetch-post.ts +++ b/starsky/starsky/clientapp/src/shared/fetch-post.ts @@ -1,24 +1,17 @@ import { IConnectionDefault } from "../interfaces/IConnectionDefault"; - +import { GetCookie } from "../get-cookie.ts"; const FetchPost = async ( url: string, body: string | FormData, method: "post" | "delete" = "post", headers: object = {} ): Promise => { - function getCookie(name: string): string { - const regex = new RegExp("(^| )" + name + "=([^;]+)"); - const match = regex.exec(document.cookie); - if (match) return match[2]; - return "X-XSRF-TOKEN"; - } - const settings: RequestInit = { method: method, body, credentials: "include" as RequestCredentials, headers: { - "X-XSRF-TOKEN": getCookie("X-XSRF-TOKEN"), + "X-XSRF-TOKEN": GetCookie("X-XSRF-TOKEN"), Accept: "application/json", ...headers } From 537c4434f11f0f37bf8f4b3e723e85c84056e5ca Mon Sep 17 00:00:00 2001 From: Dion Date: Mon, 8 Jan 2024 08:28:28 +0100 Subject: [PATCH 13/20] move cookie code and test --- .../src/shared/cookie/get-cookie.spec.ts | 42 +++++++++++++++++++ .../src/{ => shared/cookie}/get-cookie.ts | 2 +- .../clientapp/src/shared/fetch-post.spec.ts | 8 ++-- .../clientapp/src/shared/fetch-post.ts | 2 +- 4 files changed, 48 insertions(+), 6 deletions(-) create mode 100644 starsky/starsky/clientapp/src/shared/cookie/get-cookie.spec.ts rename starsky/starsky/clientapp/src/{ => shared/cookie}/get-cookie.ts (88%) diff --git a/starsky/starsky/clientapp/src/shared/cookie/get-cookie.spec.ts b/starsky/starsky/clientapp/src/shared/cookie/get-cookie.spec.ts new file mode 100644 index 0000000000..0ef3ebd7eb --- /dev/null +++ b/starsky/starsky/clientapp/src/shared/cookie/get-cookie.spec.ts @@ -0,0 +1,42 @@ +import { GetCookie } from "./get-cookie"; + +describe("GetCookie function", () => { + // Test case 1: Cookie with the specified name exists + test("should return the value of the cookie when it exists", () => { + // Mock document.cookie to simulate the presence of the cookie + document.cookie = "yourCookieName=yourCookieValue; path=/"; + + // Call the function and assert the result + expect(GetCookie("yourCookieName")).toBe("yourCookieValue"); + }); + + // Test case 2: Cookie with the specified name does not exist + test('should return "" when the cookie does not exist', () => { + // Mock document.cookie to simulate an empty cookie + document.cookie = ""; + + // Call the function and assert the result + expect(GetCookie("nonExistentCookie")).toBe(""); + }); + + // Test case 3: Cookie with the specified name has an empty value + test("should return an empty string when the cookie value is empty", () => { + // Mock document.cookie to simulate a cookie with an empty value + document.cookie = "emptyCookie=; path=/"; + + // Call the function and assert the result + expect(GetCookie("emptyCookie")).toBe(""); + }); + + // Test case 4: Cookie with the specified name has special characters + test("should return the correct value when the cookie value has special characters", () => { + // Mock document.cookie to simulate a cookie with special characters + document.cookie = + "specialCookie=%24%25%5E%26%2A%28%29%3D%2B%7B%7D%5B%5D%3B%3A%40%23%21%7C%3C%3E%2C%2F%3F; path=/"; + + // Call the function and assert the result + expect(GetCookie("specialCookie")).toBe( + "%24%25%5E%26%2A%28%29%3D%2B%7B%7D%5B%5D%3B%3A%40%23%21%7C%3C%3E%2C%2F%3F" + ); + }); +}); diff --git a/starsky/starsky/clientapp/src/get-cookie.ts b/starsky/starsky/clientapp/src/shared/cookie/get-cookie.ts similarity index 88% rename from starsky/starsky/clientapp/src/get-cookie.ts rename to starsky/starsky/clientapp/src/shared/cookie/get-cookie.ts index e11015caf0..2cfe72771e 100644 --- a/starsky/starsky/clientapp/src/get-cookie.ts +++ b/starsky/starsky/clientapp/src/shared/cookie/get-cookie.ts @@ -2,5 +2,5 @@ export function GetCookie(name: string): string { const regex = new RegExp("(^| )" + name + "=([^;]+)"); const match = regex.exec(document.cookie); if (match) return match[2]; - return "X-XSRF-TOKEN"; + return ""; } diff --git a/starsky/starsky/clientapp/src/shared/fetch-post.spec.ts b/starsky/starsky/clientapp/src/shared/fetch-post.spec.ts index f2ec4c2a11..60fb6dea95 100644 --- a/starsky/starsky/clientapp/src/shared/fetch-post.spec.ts +++ b/starsky/starsky/clientapp/src/shared/fetch-post.spec.ts @@ -15,7 +15,7 @@ describe("fetch-post", () => { headers: { Accept: "application/json", "Content-Type": "application/x-www-form-urlencoded", - "X-XSRF-TOKEN": "X-XSRF-TOKEN" + "X-XSRF-TOKEN": "" }, method: "post" }); @@ -36,7 +36,7 @@ describe("fetch-post", () => { headers: { Accept: "application/json", "Content-Type": "application/x-www-form-urlencoded", - "X-XSRF-TOKEN": "X-XSRF-TOKEN" + "X-XSRF-TOKEN": "" }, method: "post" }); @@ -62,7 +62,7 @@ describe("fetch-post", () => { headers: { Accept: "application/json", "Content-Type": "application/x-www-form-urlencoded", - "X-XSRF-TOKEN": "X-XSRF-TOKEN" + "X-XSRF-TOKEN": "" }, method: "post" }); @@ -81,7 +81,7 @@ describe("fetch-post", () => { headers: { Accept: "application/json", "Content-Type": "application/x-www-form-urlencoded", - "X-XSRF-TOKEN": "X-XSRF-TOKEN" + "X-XSRF-TOKEN": "" }, method: "post" }); diff --git a/starsky/starsky/clientapp/src/shared/fetch-post.ts b/starsky/starsky/clientapp/src/shared/fetch-post.ts index e1fd82c0fc..aa90edd515 100644 --- a/starsky/starsky/clientapp/src/shared/fetch-post.ts +++ b/starsky/starsky/clientapp/src/shared/fetch-post.ts @@ -1,5 +1,5 @@ import { IConnectionDefault } from "../interfaces/IConnectionDefault"; -import { GetCookie } from "../get-cookie.ts"; +import { GetCookie } from "./cookie/get-cookie.ts"; const FetchPost = async ( url: string, body: string | FormData, From 47e5fa32a3d97566db344a092f21f558b7def8a3 Mon Sep 17 00:00:00 2001 From: Dion Date: Mon, 8 Jan 2024 08:33:24 +0100 Subject: [PATCH 14/20] add tests --- .../starsky/Controllers/HealthController.cs | 1 + .../Models/OpenTelemetrySettingsTest.cs | 44 ++++++++++++++++--- 2 files changed, 39 insertions(+), 6 deletions(-) diff --git a/starsky/starsky/Controllers/HealthController.cs b/starsky/starsky/Controllers/HealthController.cs index c3feb0cec3..c693f8ab27 100644 --- a/starsky/starsky/Controllers/HealthController.cs +++ b/starsky/starsky/Controllers/HealthController.cs @@ -168,6 +168,7 @@ private static HealthView CreateHealthEntryLog(HealthReport result) [AllowAnonymous] public IActionResult ApplicationInsights() { + // Remove when ApplicationInsights is phased out return Content(_applicationInsightsJsHelper.ScriptPlain, "application/javascript"); } diff --git a/starsky/starskytest/starsky.foundation.platform/Models/OpenTelemetrySettingsTest.cs b/starsky/starskytest/starsky.foundation.platform/Models/OpenTelemetrySettingsTest.cs index 9db29dfe3e..ac2662d3a9 100644 --- a/starsky/starskytest/starsky.foundation.platform/Models/OpenTelemetrySettingsTest.cs +++ b/starsky/starskytest/starsky.foundation.platform/Models/OpenTelemetrySettingsTest.cs @@ -13,6 +13,17 @@ public void ServiceNameDefault() Assert.AreEqual("Starsky",result); } + [TestMethod] + public void ServiceNameProperty() + { + var result = new OpenTelemetrySettings + { + ServiceName = "test-service" + }.GetServiceName(); + + Assert.AreEqual("test-service",result); + } + [TestMethod] public void GetLogsHeaderDefault() { @@ -23,8 +34,15 @@ public void GetLogsHeaderDefault() [TestMethod] public void GetLogsHeaderFallback() { - var result = new OpenTelemetrySettings{Header = "1"}.GetLogsHeader(); - Assert.AreEqual("1",result); + var result = new OpenTelemetrySettings{Header = "logs"}.GetLogsHeader(); + Assert.AreEqual("logs",result); + } + + [TestMethod] + public void GetLogsShowProperty() + { + var result = new OpenTelemetrySettings{LogsHeader = "logs"}.GetLogsHeader(); + Assert.AreEqual("logs",result); } [TestMethod] @@ -37,8 +55,15 @@ public void GetMetricsHeaderDefault() [TestMethod] public void GetMetricsHeaderFallback() { - var result = new OpenTelemetrySettings{Header = "1"}.GetMetricsHeader(); - Assert.AreEqual("1",result); + var result = new OpenTelemetrySettings{Header = "metrics"}.GetMetricsHeader(); + Assert.AreEqual("metrics",result); + } + + [TestMethod] + public void GetMetricsShowProperty() + { + var result = new OpenTelemetrySettings{MetricsHeader = "metrics"}.GetMetricsHeader(); + Assert.AreEqual("metrics",result); } [TestMethod] @@ -51,7 +76,14 @@ public void GetTracesHeaderDefault() [TestMethod] public void GetTracesHeaderFallback() { - var result = new OpenTelemetrySettings{Header = "1"}.GetTracesHeader(); - Assert.AreEqual("1",result); + var result = new OpenTelemetrySettings{Header = "traces"}.GetTracesHeader(); + Assert.AreEqual("traces",result); + } + + [TestMethod] + public void GetTracesShowProperty() + { + var result = new OpenTelemetrySettings{MetricsHeader = "traces"}.GetMetricsHeader(); + Assert.AreEqual("traces",result); } } From 5725431da8d3af3fd16a5c6931103e67ab4fa7be Mon Sep 17 00:00:00 2001 From: Dion Date: Mon, 8 Jan 2024 08:36:28 +0100 Subject: [PATCH 15/20] add test --- .../Models/AppSettingsTest.cs | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/starsky/starskytest/starsky.foundation.platform/Models/AppSettingsTest.cs b/starsky/starskytest/starsky.foundation.platform/Models/AppSettingsTest.cs index 0657ead96e..7c84008cc2 100644 --- a/starsky/starskytest/starsky.foundation.platform/Models/AppSettingsTest.cs +++ b/starsky/starskytest/starsky.foundation.platform/Models/AppSettingsTest.cs @@ -384,7 +384,7 @@ public void AppSettings_CloneToDisplay_hideSecurityItems2() } [TestMethod] - public void AppSettings_CloneToDisplay_skipSecurityItems2() + public void AppSettings_CloneToDisplay_skip_Null_SecurityItems2_OpenTelemetrySettings() { var appSettings = new AppSettings { @@ -404,6 +404,19 @@ public void AppSettings_CloneToDisplay_skipSecurityItems2() Assert.IsNull(display.OpenTelemetry.MetricsHeader); Assert.IsNull(display.OpenTelemetry.TracesHeader); } + + [TestMethod] + public void AppSettings_CloneToDisplay_skip_object_Null_SecurityItems2_OpenTelemetrySettings() + { + var appSettings = new AppSettings + { + OpenTelemetry = null + }; + + var display = appSettings.CloneToDisplay(); + + Assert.IsNull(display.OpenTelemetry); + } [TestMethod] public void AppSettings_CloneToDisplay_hideSecurityItems_PublishProfiles() From 30d87ca93492a8cd92fb19408bb5fcd061b2068d Mon Sep 17 00:00:00 2001 From: Dion Date: Mon, 8 Jan 2024 08:55:06 +0100 Subject: [PATCH 16/20] add test --- .../Helpers/OpenTelemetryExtensionTest.cs | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/starsky/starskytest/starsky.foundation.webtelemetry/Helpers/OpenTelemetryExtensionTest.cs b/starsky/starskytest/starsky.foundation.webtelemetry/Helpers/OpenTelemetryExtensionTest.cs index ec0e948741..75476c3ac3 100644 --- a/starsky/starskytest/starsky.foundation.webtelemetry/Helpers/OpenTelemetryExtensionTest.cs +++ b/starsky/starskytest/starsky.foundation.webtelemetry/Helpers/OpenTelemetryExtensionTest.cs @@ -125,4 +125,35 @@ public void FilterPathTestSkipRealtime() var context = new DefaultHttpContext { Request = { Path = "/realtime" } }; Assert.IsFalse(OpenTelemetryExtension.FilterPath(context)); } + + [TestMethod] + public void FilterPathTestHitRealtimeFail() + { + var context = new DefaultHttpContext + { + Request = { Path = "/realtime" }, + Response = { StatusCode = 500} + }; + Assert.IsTrue(OpenTelemetryExtension.FilterPath(context)); + } + + [TestMethod] + public void FilterPathTestSkipTraceApi() + { + var context = new DefaultHttpContext { Request = { Path = "/api/open-telemetry/trace" } }; + Assert.IsFalse(OpenTelemetryExtension.FilterPath(context)); + } + + [TestMethod] + public void FilterPathTestSkipIndexAuthApi() + { + var context = new DefaultHttpContext + { + Request = { Path = "/api/index" }, + Response = { StatusCode = 401} + }; + + Assert.IsFalse(OpenTelemetryExtension.FilterPath(context)); + } + } From 30661afc45e081eb8fba3b2b5ee2f4ec6352a0dc Mon Sep 17 00:00:00 2001 From: Dion Date: Mon, 8 Jan 2024 13:39:54 +0100 Subject: [PATCH 17/20] Add information about Open Telemetry --- .../docs/developer-guide/logging/opentelemetry.md | 10 ++++++++++ documentation/docs/developer-guide/logging/readme.md | 3 +++ starsky/starsky/readme.md | 2 ++ starsky/telemetry.md | 4 ++++ 4 files changed, 19 insertions(+) create mode 100644 documentation/docs/developer-guide/logging/opentelemetry.md diff --git a/documentation/docs/developer-guide/logging/opentelemetry.md b/documentation/docs/developer-guide/logging/opentelemetry.md new file mode 100644 index 0000000000..e9e18348d7 --- /dev/null +++ b/documentation/docs/developer-guide/logging/opentelemetry.md @@ -0,0 +1,10 @@ +# Open Telemetry logging + + +``` +"app__OpenTelemetry__TracesEndpoint": "https://otlp.eu01.nr-data.net:4318/v1/traces", +"app__OpenTelemetry__MetricsEndpoint": "https://otlp.eu01.nr-data.net:4318/v1/metrics", +"app__OpenTelemetry__LogsEndpoint": "https://otlp.eu01.nr-data.net:4318/v1/logs", +"app__OpenTelemetry__Header": "api-key=EXAMPLE_KEY", +"app__OpenTelemetry__ServiceName": "starsky-dev", +``` \ No newline at end of file diff --git a/documentation/docs/developer-guide/logging/readme.md b/documentation/docs/developer-guide/logging/readme.md index 0e6867c27d..abb4126ee7 100644 --- a/documentation/docs/developer-guide/logging/readme.md +++ b/documentation/docs/developer-guide/logging/readme.md @@ -4,6 +4,9 @@ sidebar_position: 20 # Logging +## Open Telemetry logging +See [Open Telemetry logging](opentelemetry.md) + ## Desktop logs For the desktop app the logs are stored in following location: diff --git a/starsky/starsky/readme.md b/starsky/starsky/readme.md index 98fd031a2a..3a12ad32e6 100644 --- a/starsky/starsky/readme.md +++ b/starsky/starsky/readme.md @@ -92,6 +92,8 @@ You could use machine specific configuration files: appsettings.machinename.json 36. `GeoFilesSkipDownloadOnStartup` Skip download of GeoFiles on startup, _recommend to keep this false or null_ - _default false_ 37. `ExiftoolSkipDownloadOnStartup` Skip download of Exiftool on startup, _recommend to keep this false or null_ - _default false_ 38. `AccountRolesByEmailRegisterOverwrite` Overwrite the default role for a user by email address, _default empty list_ +39. `OpenTelemetry` See logging in an external service, _default no enabled_ see [OpenTelemetry](https://docs.qdraw.nl/docs/developer-guide/logging/opentelemetry.md) + ### Appsettings.json example ```json diff --git a/starsky/telemetry.md b/starsky/telemetry.md index a43a744f84..e647b515ad 100644 --- a/starsky/telemetry.md +++ b/starsky/telemetry.md @@ -121,3 +121,7 @@ You may re-enable telemetry if you'd like to re-join the program by running the `app__EnablePackageTelemetry=true` +## OpenTelemetry + +By default OpenTelemetry is disabled to collect telemetry data. +But you can setup your own provider to collect information From fc46ce4a4a4dfc75687c49d017ad96af944b5705 Mon Sep 17 00:00:00 2001 From: Dion Date: Mon, 8 Jan 2024 16:29:28 +0100 Subject: [PATCH 18/20] add default compare --- .../developer-guide/logging/opentelemetry.md | 90 ++++++++++++++++++- .../Helpers/AppSettingsCompareHelper.cs | 12 ++- starsky/starsky/appsettings.json | 5 ++ .../Helpers/AppSettingsCompareHelperTest.cs | 30 +++++++ 4 files changed, 132 insertions(+), 5 deletions(-) diff --git a/documentation/docs/developer-guide/logging/opentelemetry.md b/documentation/docs/developer-guide/logging/opentelemetry.md index e9e18348d7..c42850a601 100644 --- a/documentation/docs/developer-guide/logging/opentelemetry.md +++ b/documentation/docs/developer-guide/logging/opentelemetry.md @@ -1,10 +1,98 @@ # Open Telemetry logging +- Scroll down to [setup](#setup) for the settings +## Why OpenTelemetry? + +OpenTelemetry has emerged as a powerful and +standardized solution to address the challenges of observability in modern, +distributed systems. + +OpenTelemetry is an open-source project that provides a set of APIs, libraries, agents, +and instrumentation to enable observability in cloud-native applications. + +Observability, in the context of software systems, +refers to the ability to understand and measure how well a system is operating. +It involves collecting and analyzing data related to key aspects such as traces, +metrics, and logs. OpenTelemetry focuses on two primary pillars of observability: +tracing and metrics. + +### Distributed Tracing: +OpenTelemetry allows developers to trace requests as they traverse +through various services in a distributed environment. +This helps identify performance bottlenecks, understand dependencies between services, +and diagnose issues across the entire application stack. +By providing a standardized way to instrument code and capture trace data, +OpenTelemetry simplifies the process of generating insights into the flow of requests in complex, +microservices-based architectures. + +### Metrics Collection: +Monitoring the health and performance of applications involves the collection of metrics. +OpenTelemetry provides a consistent API for capturing metrics from different parts of an application. +Metrics such as latency, error rates, and resource utilization +can be collected and analyzed to gain a comprehensive view of application behavior. +This information is invaluable for proactive issue detection, capacity planning, and overall system optimization. + +OpenTelemetry supports multiple programming languages and integrates seamlessly with various observability backends, +including popular solutions like Prometheus, Jaeger, and Grafana. + +Its adaptability and community-driven development make it a versatile choice for organizations seeking +to implement observability in their applications, regardless of the technology stack they use. + +In conclusion, OpenTelemetry plays a pivotal role in the modern software development landscape by providing +a standardized and extensible framework for observability. +Its ability to capture distributed traces and metrics empowers developers and operators to +gain deep insights into the performance and behavior of their applications, +facilitating efficient troubleshooting and continuous improvement. + +## Setup +The following settings can be used: + +### json configuration + +use for example this file name: appsettings.machinename.json + +### Order of appsettings patch files + +1. You can use `appsettings.json` inside the application folder to set base settings. + The order of this files is used to get the values from the appsettings + - `/bin/Debug/net6.0/appsettings.patch.json` + - `/bin/Debug/net6.0/appsettings.default.json` + - `/bin/Debug/net6.0/appsettings.computername.patch.json` + - `/bin/Debug/net6.0/appsettings.json` + - `/bin/Debug/net6.0/appsettings.computername.json` + +```json +{ + "app" : { + "OpenTelemetry": { + "TracesEndpoint": "http://test", + "MetricsEndpoint": null, + "LogsEndpoint": null + } + } +} ``` + + +### Environment variables + +```bash "app__OpenTelemetry__TracesEndpoint": "https://otlp.eu01.nr-data.net:4318/v1/traces", "app__OpenTelemetry__MetricsEndpoint": "https://otlp.eu01.nr-data.net:4318/v1/metrics", "app__OpenTelemetry__LogsEndpoint": "https://otlp.eu01.nr-data.net:4318/v1/logs", "app__OpenTelemetry__Header": "api-key=EXAMPLE_KEY", "app__OpenTelemetry__ServiceName": "starsky-dev", -``` \ No newline at end of file +``` + +### Use a valid url +The properties assume that the Url is valid. It should start with http or https + +``` +Unhandled exception. System.UriFormatException: Invalid URI: The format of the URI could not be determined. +at System.Uri.CreateThis(String uri, Boolean dontEscape, UriKind uriKind, UriCreationOptions& creationOptions) +at System.Uri..ctor(String uriString) +at starsky.foundation.webtelemetry.Extensions.OpenTelemetryExtension.<>c__DisplayClass0_0. +b__4(OtlpExporterOptions o) in ..Extensions/OpenTelemetryExtension.cs:line 48 +``` + diff --git a/starsky/starsky.foundation.platform/Helpers/AppSettingsCompareHelper.cs b/starsky/starsky.foundation.platform/Helpers/AppSettingsCompareHelper.cs index 0b4180bb3b..7cc9b5aa0b 100644 --- a/starsky/starsky.foundation.platform/Helpers/AppSettingsCompareHelper.cs +++ b/starsky/starsky.foundation.platform/Helpers/AppSettingsCompareHelper.cs @@ -43,9 +43,9 @@ private static void CompareMultipleObjects(PropertyInfo propertyB, PropertyInfo { if (propertyInfoFromA.PropertyType == typeof(OpenTelemetrySettings) && propertyB.PropertyType == typeof(OpenTelemetrySettings)) { - var oldBoolValue = (OpenTelemetrySettings?)propertyInfoFromA.GetValue(sourceIndexItem, null); - var newBoolValue = (OpenTelemetrySettings?)propertyB.GetValue(updateObject, null); - CompareOpenTelemetrySettingsObject(propertyB.Name, sourceIndexItem, oldBoolValue, newBoolValue, differenceList); + var oldObjectValue = (OpenTelemetrySettings?)propertyInfoFromA.GetValue(sourceIndexItem, null); + var newObjectValue = (OpenTelemetrySettings?)propertyB.GetValue(updateObject, null); + CompareOpenTelemetrySettingsObject(propertyB.Name, sourceIndexItem, oldObjectValue, newObjectValue, differenceList); } } @@ -55,8 +55,12 @@ private static void CompareOpenTelemetrySettingsObject(string propertyName, AppS { if ( oldKeyValuePairStringStringValue == null || newKeyValuePairStringStringValue == null || + // compare lists JsonSerializer.Serialize(oldKeyValuePairStringStringValue) == - JsonSerializer.Serialize(newKeyValuePairStringStringValue)) + JsonSerializer.Serialize(newKeyValuePairStringStringValue) || + // default options + JsonSerializer.Serialize(newKeyValuePairStringStringValue) == + JsonSerializer.Serialize(new OpenTelemetrySettings())) { return; } diff --git a/starsky/starsky/appsettings.json b/starsky/starsky/appsettings.json index bcf3f1f57a..279fb6552e 100644 --- a/starsky/starsky/appsettings.json +++ b/starsky/starsky/appsettings.json @@ -64,6 +64,11 @@ "SyncOnStartup": "true", "DemoUnsafeDeleteStorageFolder" : "false", "useSystemTrash": "true", + "OpenTelemetry": { + "TracesEndpoint": null, + "MetricsEndpoint": null, + "LogsEndpoint": null + }, "publishProfiles": { "_default": [ { diff --git a/starsky/starskytest/starsky.foundation.platform/Helpers/AppSettingsCompareHelperTest.cs b/starsky/starskytest/starsky.foundation.platform/Helpers/AppSettingsCompareHelperTest.cs index 6effbb8f0e..1673caab1c 100644 --- a/starsky/starskytest/starsky.foundation.platform/Helpers/AppSettingsCompareHelperTest.cs +++ b/starsky/starskytest/starsky.foundation.platform/Helpers/AppSettingsCompareHelperTest.cs @@ -520,5 +520,35 @@ public void OpenTelemetrySettings() Assert.AreEqual(source.OpenTelemetry.LogsHeader, to.OpenTelemetry.LogsHeader); } + + [TestMethod] + public void OpenTelemetrySettings_Ignore_DefaultOption() + { + var source = new AppSettings + { + OpenTelemetry = new OpenTelemetrySettings + { + Header = "source/test", + TracesEndpoint = "source/traces", + TracesHeader = "source/traces", + MetricsEndpoint = "source/metrics", + MetricsHeader = "source/metrics", + LogsEndpoint = "source/logs", + LogsHeader = "source/logs" + } + }; + + var to = new AppSettings + { + OpenTelemetry = new OpenTelemetrySettings() + }; + + AppSettingsCompareHelper.Compare(source, to); + + Assert.AreEqual(source.OpenTelemetry.Header, "source/test"); + Assert.AreEqual(source.OpenTelemetry.TracesEndpoint, "source/traces"); + Assert.AreEqual(source.OpenTelemetry.MetricsEndpoint, "source/metrics"); + Assert.AreEqual(source.OpenTelemetry.LogsEndpoint, "source/logs"); + } } } From 6e0a83ddac19ef03138f9311efabe17025ed64ee Mon Sep 17 00:00:00 2001 From: Dion Date: Mon, 8 Jan 2024 16:37:42 +0100 Subject: [PATCH 19/20] change order --- .../Helpers/AppSettingsCompareHelperTest.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/starsky/starskytest/starsky.foundation.platform/Helpers/AppSettingsCompareHelperTest.cs b/starsky/starskytest/starsky.foundation.platform/Helpers/AppSettingsCompareHelperTest.cs index 1673caab1c..73e14db215 100644 --- a/starsky/starskytest/starsky.foundation.platform/Helpers/AppSettingsCompareHelperTest.cs +++ b/starsky/starskytest/starsky.foundation.platform/Helpers/AppSettingsCompareHelperTest.cs @@ -545,10 +545,10 @@ public void OpenTelemetrySettings_Ignore_DefaultOption() AppSettingsCompareHelper.Compare(source, to); - Assert.AreEqual(source.OpenTelemetry.Header, "source/test"); - Assert.AreEqual(source.OpenTelemetry.TracesEndpoint, "source/traces"); - Assert.AreEqual(source.OpenTelemetry.MetricsEndpoint, "source/metrics"); - Assert.AreEqual(source.OpenTelemetry.LogsEndpoint, "source/logs"); + Assert.AreEqual("source/test", source.OpenTelemetry.Header); + Assert.AreEqual("source/traces", source.OpenTelemetry.TracesEndpoint); + Assert.AreEqual("source/metrics", source.OpenTelemetry.MetricsEndpoint); + Assert.AreEqual("source/logs", source.OpenTelemetry.LogsEndpoint); } } } From d1bcf0462fa6c7850070afb99067da5a873a8a59 Mon Sep 17 00:00:00 2001 From: Dion Date: Mon, 8 Jan 2024 16:43:41 +0100 Subject: [PATCH 20/20] keep a changelog --- history.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/history.md b/history.md index c2414ccb85..316a2497b3 100644 --- a/history.md +++ b/history.md @@ -36,7 +36,8 @@ Semantic Versioning 2.0.0 is from version 0.1.6+ ## List of versions ## version 0.5.15 _(Unreleased)_ - 2024-01-? {#v0.5.15} -- no changes yet + +- [x] (Added) _Back-end_ Add support for OpenTelemetry (server side only) (PR #1323) ## version 0.5.14 - 2023-12-29 {#v0.5.14}