From f47798b9a927d8f7bc1a8c8e0a0a1cf798fb53a8 Mon Sep 17 00:00:00 2001 From: Russ Cam Date: Tue, 25 May 2021 10:28:11 +1000 Subject: [PATCH 01/21] Fix traceparent header docs typo (#1304) Fixes #1285 --- docs/configuration.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/configuration.asciidoc b/docs/configuration.asciidoc index 34dabb291..71d626c82 100644 --- a/docs/configuration.asciidoc +++ b/docs/configuration.asciidoc @@ -879,7 +879,7 @@ When this setting is `true`, the agent also adds the header `elasticapm-tracepar [options="header"] |============ | Environment variable name | IConfiguration or Web.config key -| `ELASTIC_APM_USE_ELASTIC_TRACEPARENT_HEADER` | `ElasticApm:UseElasticTraceparentHeder` +| `ELASTIC_APM_USE_ELASTIC_TRACEPARENT_HEADER` | `ElasticApm:UseElasticTraceparentHeader` |============ [options="header"] From bd080ee91b32f13eb8331b47201e1813ff1e715c Mon Sep 17 00:00:00 2001 From: Russ Cam Date: Wed, 26 May 2021 11:43:14 +1000 Subject: [PATCH 02/21] Use storage account in destination.service.resource (#1284) Relates: elastic/apm#426 --- .../AzureBlobStorageDiagnosticListener.cs | 2 +- ...AzureFileShareStorageDiagnosticListener.cs | 16 +- .../AzureQueueStorageDiagnosticListener.cs | 24 +- src/Elastic.Apm.Azure.Storage/BlobUrl.cs | 22 +- .../MicrosoftAzureBlobStorageTracer.cs | 3 +- src/Elastic.Apm.MongoDb/LICENSE | 417 +++++++++--------- ...FileShareStorageDiagnosticListenerTests.cs | 2 +- .../BlobStorageTestsBase.cs | 2 +- 8 files changed, 239 insertions(+), 249 deletions(-) diff --git a/src/Elastic.Apm.Azure.Storage/AzureBlobStorageDiagnosticListener.cs b/src/Elastic.Apm.Azure.Storage/AzureBlobStorageDiagnosticListener.cs index 5bcd047e6..8f326722e 100644 --- a/src/Elastic.Apm.Azure.Storage/AzureBlobStorageDiagnosticListener.cs +++ b/src/Elastic.Apm.Azure.Storage/AzureBlobStorageDiagnosticListener.cs @@ -212,7 +212,7 @@ private void OnStart(KeyValuePair kv, string action) Service = new Destination.DestinationService { Name = AzureBlobStorage.SubType, - Resource = $"{AzureBlobStorage.SubType}/{blobUrl.ResourceName}", + Resource = $"{AzureBlobStorage.SubType}/{blobUrl.StorageAccountName}", Type = ApiConstants.TypeStorage } }; diff --git a/src/Elastic.Apm.Azure.Storage/AzureFileShareStorageDiagnosticListener.cs b/src/Elastic.Apm.Azure.Storage/AzureFileShareStorageDiagnosticListener.cs index c5686d671..692cd30f5 100644 --- a/src/Elastic.Apm.Azure.Storage/AzureFileShareStorageDiagnosticListener.cs +++ b/src/Elastic.Apm.Azure.Storage/AzureFileShareStorageDiagnosticListener.cs @@ -99,7 +99,7 @@ private void OnStart(KeyValuePair kv, string action) } var urlTag = activity.Tags.FirstOrDefault(t => t.Key == "url").Value; - var fileShareUrl = new FileShareUrl(urlTag); + var fileShareUrl = new FileShareUrl(new Uri(urlTag)); var spanName = $"{AzureFileStorage.SpanName} {action} {fileShareUrl.ResourceName}"; var span = currentSegment.StartSpan(spanName, ApiConstants.TypeStorage, AzureFileStorage.SubType, action); @@ -112,7 +112,7 @@ private void OnStart(KeyValuePair kv, string action) Service = new Destination.DestinationService { Name = AzureFileStorage.SubType, - Resource = $"{AzureFileStorage.SubType}/{fileShareUrl.ResourceName}", + Resource = $"{AzureFileStorage.SubType}/{fileShareUrl.StorageAccountName}", Type = ApiConstants.TypeStorage } }; @@ -175,17 +175,9 @@ private void OnException(KeyValuePair kv) segment.End(); } - private class FileShareUrl + private class FileShareUrl : StorageUrl { - public FileShareUrl(string url) - { - var builder = new UriBuilder(url); - - FullyQualifiedNamespace = builder.Uri.GetLeftPart(UriPartial.Authority) + "/"; - ResourceName = builder.Uri.AbsolutePath.TrimStart('/'); - } - - public string FullyQualifiedNamespace { get; } + public FileShareUrl(Uri url) : base(url) => ResourceName = url.AbsolutePath.TrimStart('/'); public string ResourceName { get; } } diff --git a/src/Elastic.Apm.Azure.Storage/AzureQueueStorageDiagnosticListener.cs b/src/Elastic.Apm.Azure.Storage/AzureQueueStorageDiagnosticListener.cs index 4dab872d2..311b57a86 100644 --- a/src/Elastic.Apm.Azure.Storage/AzureQueueStorageDiagnosticListener.cs +++ b/src/Elastic.Apm.Azure.Storage/AzureQueueStorageDiagnosticListener.cs @@ -97,9 +97,9 @@ private void OnSendStart(KeyValuePair kv) string destinationAddress = null; var urlTag = activity.Tags.FirstOrDefault(t => t.Key == "url").Value; - if (!string.IsNullOrEmpty(urlTag)) + if (!string.IsNullOrEmpty(urlTag) && Uri.TryCreate(urlTag, UriKind.Absolute, out var url)) { - var queueUrl = new QueueUrl(urlTag); + var queueUrl = new QueueUrl(url); queueName = queueUrl.QueueName; destinationAddress = queueUrl.FullyQualifiedNamespace; } @@ -146,8 +146,8 @@ private void OnReceiveStart(KeyValuePair kv) } var urlTag = activity.Tags.FirstOrDefault(t => t.Key == "url").Value; - var queueName = !string.IsNullOrEmpty(urlTag) - ? new QueueUrl(urlTag).QueueName + var queueName = !string.IsNullOrEmpty(urlTag) && Uri.TryCreate(urlTag, UriKind.Absolute, out var url) + ? new QueueUrl(url).QueueName : null; if (MatchesIgnoreMessageQueues(queueName)) @@ -243,20 +243,12 @@ private void OnException(KeyValuePair kv) /// /// Working with a queue url to extract the queue name and address. /// - private class QueueUrl + private class QueueUrl : StorageUrl { - public QueueUrl(string url) - { - var builder = new UriBuilder(url); - - FullyQualifiedNamespace = builder.Uri.GetLeftPart(UriPartial.Authority) + "/"; - - QueueName = builder.Uri.Segments.Length > 1 - ? builder.Uri.Segments[1].TrimEnd('/') + public QueueUrl(Uri url) : base(url) => + QueueName = url.Segments.Length > 1 + ? url.Segments[1].TrimEnd('/') : null; - } - - public string FullyQualifiedNamespace { get; } public string QueueName { get; } } diff --git a/src/Elastic.Apm.Azure.Storage/BlobUrl.cs b/src/Elastic.Apm.Azure.Storage/BlobUrl.cs index 2bcb62a15..cdd419baa 100644 --- a/src/Elastic.Apm.Azure.Storage/BlobUrl.cs +++ b/src/Elastic.Apm.Azure.Storage/BlobUrl.cs @@ -7,22 +7,28 @@ namespace Elastic.Apm.Azure.Storage { - internal class BlobUrl + internal class BlobUrl : StorageUrl { - public BlobUrl(Uri url) - { - var builder = new UriBuilder(url); - - FullyQualifiedNamespace = builder.Uri.GetLeftPart(UriPartial.Authority) + "/"; - ResourceName = builder.Uri.AbsolutePath.TrimStart('/'); - } + public BlobUrl(Uri url) : base(url) => ResourceName = url.AbsolutePath.TrimStart('/'); public BlobUrl(string url) : this(new Uri(url)) { } public string ResourceName { get; } + } + + internal abstract class StorageUrl + { + private static char[] SplitDomain = { '.' }; + + protected StorageUrl(Uri url) + { + StorageAccountName = url.Host.Split(SplitDomain, 2)[0]; + FullyQualifiedNamespace = url.GetLeftPart(UriPartial.Authority) + "/"; + } + public string StorageAccountName { get; } public string FullyQualifiedNamespace { get; } } } diff --git a/src/Elastic.Apm.Azure.Storage/MicrosoftAzureBlobStorageTracer.cs b/src/Elastic.Apm.Azure.Storage/MicrosoftAzureBlobStorageTracer.cs index 0f57fee81..a6ba95202 100644 --- a/src/Elastic.Apm.Azure.Storage/MicrosoftAzureBlobStorageTracer.cs +++ b/src/Elastic.Apm.Azure.Storage/MicrosoftAzureBlobStorageTracer.cs @@ -109,7 +109,6 @@ public ISpan StartSpan(IApmAgent agent, string method, Uri requestUrl, Func Date: Wed, 26 May 2021 11:44:38 +1000 Subject: [PATCH 03/21] Do not capture HTTP child spans for Elasticsearch (#1306) This commit updates the Elasticsearch integration to not capture HTTP child spans for calls to Elasticsearch. An instrumentation flag is set by spans started by the Elasticsearch integration diagnostic listeners that is checked in the HttpDiagnosticListener. We might consider refactoring to include an Exit property on span in the future, that can be checked. Closes #1276 --- .../ElasticsearchDiagnosticsListenerBase.cs | 3 +- .../HttpDiagnosticListenerImplBase.cs | 5 +- src/Elastic.Apm/Model/InstrumentationFlag.cs | 3 +- .../Elastic.Apm.Elasticsearch.Tests.csproj | 4 +- .../ElasticsearchFixture.cs | 41 ++++++++++++++ .../ElasticsearchTestContainer.cs | 17 ++++++ ...ElasticsearchTestContainerConfiguration.cs | 53 +++++++++++++++++++ .../ElasticsearchTests.cs | 47 ++++++++++++++++ 8 files changed, 167 insertions(+), 6 deletions(-) create mode 100644 test/Elastic.Apm.Elasticsearch.Tests/ElasticsearchFixture.cs create mode 100644 test/Elastic.Apm.Elasticsearch.Tests/ElasticsearchTestContainer.cs create mode 100644 test/Elastic.Apm.Elasticsearch.Tests/ElasticsearchTestContainerConfiguration.cs create mode 100644 test/Elastic.Apm.Elasticsearch.Tests/ElasticsearchTests.cs diff --git a/src/Elastic.Apm.Elasticsearch/ElasticsearchDiagnosticsListenerBase.cs b/src/Elastic.Apm.Elasticsearch/ElasticsearchDiagnosticsListenerBase.cs index ce4d94570..a5d4f6653 100644 --- a/src/Elastic.Apm.Elasticsearch/ElasticsearchDiagnosticsListenerBase.cs +++ b/src/Elastic.Apm.Elasticsearch/ElasticsearchDiagnosticsListenerBase.cs @@ -47,7 +47,7 @@ internal bool TryStartElasticsearchSpan(string name, out Span span, Uri instance { span = null; var transaction = ApmAgent.Tracer.CurrentTransaction; - if (transaction == null) + if (transaction is null) return false; span = (Span)ApmAgent.GetCurrentExecutionSegment() @@ -56,6 +56,7 @@ internal bool TryStartElasticsearchSpan(string name, out Span span, Uri instance ApiConstants.TypeDb, ApiConstants.SubtypeElasticsearch); + span.InstrumentationFlag = InstrumentationFlag.Elasticsearch; span.Action = name; SetDbContext(span, instanceUri); SetDestination(span, instanceUri); diff --git a/src/Elastic.Apm/DiagnosticListeners/HttpDiagnosticListenerImplBase.cs b/src/Elastic.Apm/DiagnosticListeners/HttpDiagnosticListenerImplBase.cs index 292e59af3..02de5f240 100644 --- a/src/Elastic.Apm/DiagnosticListeners/HttpDiagnosticListenerImplBase.cs +++ b/src/Elastic.Apm/DiagnosticListeners/HttpDiagnosticListenerImplBase.cs @@ -126,9 +126,8 @@ private void ProcessStartEvent(TRequest request, Uri requestUrl) { if (_realAgent?.TracerInternal.CurrentSpan is Span currentSpan) { - // if there's a current span that has been instrumented for Azure, don't create a span for - // the current request - if (currentSpan.InstrumentationFlag == InstrumentationFlag.Azure) + // if the current span is an exit span, don't create a span for the current request + if (currentSpan.InstrumentationFlag == InstrumentationFlag.Azure || currentSpan.InstrumentationFlag == InstrumentationFlag.Elasticsearch) return; } diff --git a/src/Elastic.Apm/Model/InstrumentationFlag.cs b/src/Elastic.Apm/Model/InstrumentationFlag.cs index c964b2cee..eebfccc1d 100644 --- a/src/Elastic.Apm/Model/InstrumentationFlag.cs +++ b/src/Elastic.Apm/Model/InstrumentationFlag.cs @@ -23,6 +23,7 @@ internal enum InstrumentationFlag : short EfClassic = 1 << 3, SqlClient = 1 << 4, AspNetClassic = 1 << 5, - Azure = 1 << 6 + Azure = 1 << 6, + Elasticsearch = 1 << 7, } } diff --git a/test/Elastic.Apm.Elasticsearch.Tests/Elastic.Apm.Elasticsearch.Tests.csproj b/test/Elastic.Apm.Elasticsearch.Tests/Elastic.Apm.Elasticsearch.Tests.csproj index 79e37b08d..49a20a5e4 100644 --- a/test/Elastic.Apm.Elasticsearch.Tests/Elastic.Apm.Elasticsearch.Tests.csproj +++ b/test/Elastic.Apm.Elasticsearch.Tests/Elastic.Apm.Elasticsearch.Tests.csproj @@ -17,7 +17,9 @@ - + + + diff --git a/test/Elastic.Apm.Elasticsearch.Tests/ElasticsearchFixture.cs b/test/Elastic.Apm.Elasticsearch.Tests/ElasticsearchFixture.cs new file mode 100644 index 000000000..000490cc5 --- /dev/null +++ b/test/Elastic.Apm.Elasticsearch.Tests/ElasticsearchFixture.cs @@ -0,0 +1,41 @@ +// Licensed to Elasticsearch B.V under +// one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System; +using System.Threading.Tasks; +using DotNet.Testcontainers.Containers.Builders; +using Xunit; + +namespace Elastic.Apm.Elasticsearch.Tests +{ + public class ElasticsearchFixture : IDisposable, IAsyncLifetime + { + private readonly ElasticsearchTestContainer _container; + + public ElasticsearchFixture() + { + var containerBuilder = new TestcontainersBuilder() + .WithElasticsearch(new ElasticsearchTestContainerConfiguration()); + + _container = containerBuilder.Build(); + } + + public string ConnectionString { get; private set; } + + public async Task InitializeAsync() + { + await _container.StartAsync(); + ConnectionString = _container.ConnectionString; + } + + public async Task DisposeAsync() + { + await _container.StopAsync(); + _container.Dispose(); + } + + public void Dispose() => _container?.Dispose(); + } +} diff --git a/test/Elastic.Apm.Elasticsearch.Tests/ElasticsearchTestContainer.cs b/test/Elastic.Apm.Elasticsearch.Tests/ElasticsearchTestContainer.cs new file mode 100644 index 000000000..562a2f44c --- /dev/null +++ b/test/Elastic.Apm.Elasticsearch.Tests/ElasticsearchTestContainer.cs @@ -0,0 +1,17 @@ +// Licensed to Elasticsearch B.V under +// one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using DotNet.Testcontainers.Containers.Configurations; +using DotNet.Testcontainers.Containers.Modules.Abstractions; + +namespace Elastic.Apm.Elasticsearch.Tests +{ + public class ElasticsearchTestContainer : HostedServiceContainer + { + internal ElasticsearchTestContainer(TestcontainersConfiguration configuration) : base(configuration) => Hostname = "localhost"; + + public override string ConnectionString => $"http://{Hostname}:{Port}"; + } +} diff --git a/test/Elastic.Apm.Elasticsearch.Tests/ElasticsearchTestContainerConfiguration.cs b/test/Elastic.Apm.Elasticsearch.Tests/ElasticsearchTestContainerConfiguration.cs new file mode 100644 index 000000000..ce1d3c057 --- /dev/null +++ b/test/Elastic.Apm.Elasticsearch.Tests/ElasticsearchTestContainerConfiguration.cs @@ -0,0 +1,53 @@ +// Licensed to Elasticsearch B.V under +// one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System.Linq; +using System.Threading.Tasks; +using DotNet.Testcontainers.Client; +using DotNet.Testcontainers.Containers.Builders; +using DotNet.Testcontainers.Containers.Configurations.Abstractions; +using DotNet.Testcontainers.Containers.OutputConsumers; +using DotNet.Testcontainers.Containers.WaitStrategies; + +namespace Elastic.Apm.Elasticsearch.Tests +{ + public static class TestcontainersBuilderExtensions + { + public static ITestcontainersBuilder WithElasticsearch( + this ITestcontainersBuilder builder, + ElasticsearchTestContainerConfiguration configuration + ) + { + builder = configuration.Environments.Aggregate(builder, (current, environment) => + current.WithEnvironment(environment.Key, environment.Value)); + + return builder + .WithImage(configuration.Image) + .WithPortBinding(configuration.Port, configuration.DefaultPort) + .WithWaitStrategy(configuration.WaitStrategy) + .ConfigureContainer(container => + { + container.Port = configuration.DefaultPort; + }); + } + } + + public class ElasticsearchTestContainerConfiguration : HostedServiceConfiguration + { + private const int ElasticsearchDefaultPort = 9200; + private const string ElasticsearchImageVersion = "7.12.1"; + + public ElasticsearchTestContainerConfiguration() + : this($"docker.elastic.co/elasticsearch/elasticsearch:{ElasticsearchImageVersion}") { } + + public ElasticsearchTestContainerConfiguration(string image) : base(image, ElasticsearchDefaultPort) + { + Environments["discovery.type"] = "single-node"; + WaitStrategy = Wait.UntilBashCommandsAreCompleted("curl -s -k http://localhost:9200/_cluster/health | grep -vq '\"status\":\"\\(^red\\)\"'"); + } + + public override IWaitUntil WaitStrategy { get; } + } +} diff --git a/test/Elastic.Apm.Elasticsearch.Tests/ElasticsearchTests.cs b/test/Elastic.Apm.Elasticsearch.Tests/ElasticsearchTests.cs new file mode 100644 index 000000000..a3ac35755 --- /dev/null +++ b/test/Elastic.Apm.Elasticsearch.Tests/ElasticsearchTests.cs @@ -0,0 +1,47 @@ +// Licensed to Elasticsearch B.V under +// one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System; +using System.Threading.Tasks; +using Elastic.Apm.Api; +using Elastic.Apm.DiagnosticSource; +using Elastic.Apm.Tests.Utilities; +using Elasticsearch.Net; +using Elasticsearch.Net.VirtualizedCluster; +using FluentAssertions; +using Xunit; + +namespace Elastic.Apm.Elasticsearch.Tests +{ + public class ElasticsearchTests : IClassFixture + { + private readonly ElasticLowLevelClient _client; + + public ElasticsearchTests(ElasticsearchFixture fixture) + { + var settings = new ConnectionConfiguration(new Uri(fixture.ConnectionString)); + _client = new ElasticLowLevelClient(settings); + } + + [Fact] + public async Task Elasticsearch_Span_Does_Not_Have_Http_Child_Span() + { + var payloadSender = new MockPayloadSender(); + using (var agent = new ApmAgent(new TestAgentComponents(payloadSender: payloadSender))) + using (agent.Subscribe(new ElasticsearchDiagnosticsSubscriber(), new HttpDiagnosticsSubscriber())) + { + var searchResponse = await agent.Tracer.CaptureTransaction("Call Client", ApiConstants.ActionExec, + async () => await _client.SearchAsync(PostData.Empty) + ); + searchResponse.Should().NotBeNull(); + searchResponse.Success.Should().BeTrue(); + searchResponse.AuditTrail.Should().NotBeEmpty(); + + var spans = payloadSender.SpansOnFirstTransaction; + spans.Should().NotBeEmpty().And.NotContain(s => s.Subtype == ApiConstants.SubtypeHttp); + } + } + } +} From 4591f02a53de6104c80fd3c56bce640e14c80602 Mon Sep 17 00:00:00 2001 From: Gergely Kalapos Date: Wed, 26 May 2021 11:04:55 +0200 Subject: [PATCH 04/21] Fix SanitizeFieldNamesTests (#1299) * Update SanitizeFieldNamesTests.cs * Fix failing sanitize tests --- src/Elastic.Apm/Filters/ErrorContextSanitizerFilter.cs | 2 +- src/Elastic.Apm/Filters/HeaderDictionarySanitizerFilter.cs | 2 +- test/Elastic.Apm.AspNetCore.Tests/SanitizeFieldNamesTests.cs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Elastic.Apm/Filters/ErrorContextSanitizerFilter.cs b/src/Elastic.Apm/Filters/ErrorContextSanitizerFilter.cs index 089edff61..8e2409763 100644 --- a/src/Elastic.Apm/Filters/ErrorContextSanitizerFilter.cs +++ b/src/Elastic.Apm/Filters/ErrorContextSanitizerFilter.cs @@ -22,7 +22,7 @@ public IError Filter(IError error) { if (realError.Context.Request?.Headers != null && realError.ConfigSnapshot != null) { - foreach (var key in realError.Context?.Request?.Headers?.Keys) + foreach (var key in realError.Context?.Request?.Headers?.Keys.ToList()) { if (WildcardMatcher.IsAnyMatch(realError.ConfigSnapshot.SanitizeFieldNames, key)) realError.Context.Request.Headers[key] = Consts.Redacted; diff --git a/src/Elastic.Apm/Filters/HeaderDictionarySanitizerFilter.cs b/src/Elastic.Apm/Filters/HeaderDictionarySanitizerFilter.cs index 729b641f4..d3b570ea4 100644 --- a/src/Elastic.Apm/Filters/HeaderDictionarySanitizerFilter.cs +++ b/src/Elastic.Apm/Filters/HeaderDictionarySanitizerFilter.cs @@ -22,7 +22,7 @@ public ITransaction Filter(ITransaction transaction) { if (realTransaction.IsContextCreated && realTransaction.Context.Request?.Headers != null) { - foreach (var key in realTransaction.Context?.Request?.Headers?.Keys) + foreach (var key in realTransaction.Context?.Request?.Headers?.Keys.ToList()) { if (WildcardMatcher.IsAnyMatch(realTransaction.ConfigSnapshot.SanitizeFieldNames, key)) realTransaction.Context.Request.Headers[key] = Consts.Redacted; diff --git a/test/Elastic.Apm.AspNetCore.Tests/SanitizeFieldNamesTests.cs b/test/Elastic.Apm.AspNetCore.Tests/SanitizeFieldNamesTests.cs index b55faf533..e89901241 100644 --- a/test/Elastic.Apm.AspNetCore.Tests/SanitizeFieldNamesTests.cs +++ b/test/Elastic.Apm.AspNetCore.Tests/SanitizeFieldNamesTests.cs @@ -320,7 +320,7 @@ public async Task SanitizeHeadersOnError(string headerName, bool useOnlyDiagnost _capturedPayload.FirstTransaction.Context.Request.Headers[headerName].Should().Be("[REDACTED]"); _capturedPayload.WaitForErrors(); - _capturedPayload.Errors.Should().ContainSingle(); + _capturedPayload.Errors.Should().NotBeEmpty(); _capturedPayload.FirstError.Context.Should().NotBeNull(); _capturedPayload.FirstError.Context.Request.Should().NotBeNull(); _capturedPayload.FirstError.Context.Request.Headers.Should().NotBeNull(); From a986d1882428966f6ff628caad868412db986106 Mon Sep 17 00:00:00 2001 From: Russ Cam Date: Thu, 27 May 2021 10:52:32 +1000 Subject: [PATCH 05/21] Create receive messaging span when inside transaction (#1308) This commit updates the Azure Service Bus integrations to create messaging spans for receiving messages from a queue or subscription when inside a traced transaction. --- ...reMessagingServiceBusDiagnosticListener.cs | 23 +++++++++++---- ...rosoftAzureServiceBusDiagnosticListener.cs | 21 ++++++++++---- ...sagingServiceBusDiagnosticListenerTests.cs | 28 +++++++++++++++++++ ...tAzureServiceBusDiagnosticListenerTests.cs | 27 ++++++++++++++++++ 4 files changed, 88 insertions(+), 11 deletions(-) diff --git a/src/Elastic.Apm.Azure.ServiceBus/AzureMessagingServiceBusDiagnosticListener.cs b/src/Elastic.Apm.Azure.ServiceBus/AzureMessagingServiceBusDiagnosticListener.cs index 092b82d37..80df12d07 100644 --- a/src/Elastic.Apm.Azure.ServiceBus/AzureMessagingServiceBusDiagnosticListener.cs +++ b/src/Elastic.Apm.Azure.ServiceBus/AzureMessagingServiceBusDiagnosticListener.cs @@ -101,18 +101,29 @@ private void OnReceiveStart(KeyValuePair kv, string action) ? $"{ServiceBus.SegmentName} {action}" : $"{ServiceBus.SegmentName} {action} from {queueName}"; - var transaction = ApmAgent.Tracer.StartTransaction(transactionName, ApiConstants.TypeMessaging); - transaction.Context.Service = new Service(null, null) { Framework = _framework }; + IExecutionSegment segment; + if (ApmAgent.Tracer.CurrentTransaction is null) + { + var transaction = ApmAgent.Tracer.StartTransaction(transactionName, ApiConstants.TypeMessaging); + transaction.Context.Service = new Service(null, null) { Framework = _framework }; + segment = transaction; + } + else + { + var span = ApmAgent.GetCurrentExecutionSegment().StartSpan(transactionName, ApiConstants.TypeMessaging, ServiceBus.SubType, action); + segment = span; + } // transaction creation will create an activity, so use this as the key. var activityId = Activity.Current.Id; - if (!_processingSegments.TryAdd(activityId, transaction)) + if (!_processingSegments.TryAdd(activityId, segment)) { - Logger.Error()?.Log( - "Could not add {Action} transaction {TransactionId} for activity {ActivityId} to tracked segments", + Logger.Trace()?.Log( + "Could not add {Action} {SegmentName} {TransactionId} for activity {ActivityId} to tracked segments", action, - transaction.Id, + segment is ITransaction ? "transaction" : "span", + segment.Id, activityId); } } diff --git a/src/Elastic.Apm.Azure.ServiceBus/MicrosoftAzureServiceBusDiagnosticListener.cs b/src/Elastic.Apm.Azure.ServiceBus/MicrosoftAzureServiceBusDiagnosticListener.cs index b45f980b7..3f9bddd59 100644 --- a/src/Elastic.Apm.Azure.ServiceBus/MicrosoftAzureServiceBusDiagnosticListener.cs +++ b/src/Elastic.Apm.Azure.ServiceBus/MicrosoftAzureServiceBusDiagnosticListener.cs @@ -99,18 +99,29 @@ private void OnReceiveStart(KeyValuePair kv, string action, Prop ? $"{ServiceBus.SegmentName} {action}" : $"{ServiceBus.SegmentName} {action} from {queueName}"; - var transaction = ApmAgent.Tracer.StartTransaction(transactionName, ApiConstants.TypeMessaging); - transaction.Context.Service = new Service(null, null) { Framework = _framework }; + IExecutionSegment segment; + if (ApmAgent.Tracer.CurrentTransaction is null) + { + var transaction = ApmAgent.Tracer.StartTransaction(transactionName, ApiConstants.TypeMessaging); + transaction.Context.Service = new Service(null, null) { Framework = _framework }; + segment = transaction; + } + else + { + var span = ApmAgent.GetCurrentExecutionSegment().StartSpan(transactionName, ApiConstants.TypeMessaging, ServiceBus.SubType, action); + segment = span; + } // transaction creation will create an activity, so use this as the key. var activityId = Activity.Current.Id; - if (!_processingSegments.TryAdd(activityId, transaction)) + if (!_processingSegments.TryAdd(activityId, segment)) { Logger.Trace()?.Log( - "Could not add {Action} transaction {TransactionId} for activity {ActivityId} to tracked segments", + "Could not add {Action} {SegmentName} {TransactionId} for activity {ActivityId} to tracked segments", action, - transaction.Id, + segment is ITransaction ? "transaction" : "span", + segment.Id, activityId); } } diff --git a/test/Elastic.Apm.Azure.ServiceBus.Tests/AzureMessagingServiceBusDiagnosticListenerTests.cs b/test/Elastic.Apm.Azure.ServiceBus.Tests/AzureMessagingServiceBusDiagnosticListenerTests.cs index 4c45cf797..1a80763f4 100644 --- a/test/Elastic.Apm.Azure.ServiceBus.Tests/AzureMessagingServiceBusDiagnosticListenerTests.cs +++ b/test/Elastic.Apm.Azure.ServiceBus.Tests/AzureMessagingServiceBusDiagnosticListenerTests.cs @@ -1,4 +1,5 @@ using System; +using System.Linq; using System.Threading.Tasks; using Azure.Messaging.ServiceBus; using Azure.Messaging.ServiceBus.Administration; @@ -156,6 +157,33 @@ await sender.ScheduleMessageAsync( destination.Service.Type.Should().Be(ApiConstants.TypeMessaging); } + [AzureCredentialsFact] + public async Task Capture_Span_When_Receive_From_Queue_Inside_Transaction() + { + await using var scope = await QueueScope.CreateWithQueue(_adminClient); + var sender = _client.CreateSender(scope.QueueName); + var receiver = _client.CreateReceiver(scope.QueueName); + + await sender.SendMessageAsync( + new ServiceBusMessage("test message")).ConfigureAwait(false); + + await _agent.Tracer.CaptureTransaction("Receive messages", ApiConstants.TypeMessaging, async t => + { + await receiver.ReceiveMessageAsync(TimeSpan.FromSeconds(30)).ConfigureAwait(false); + }); + + + if (!_sender.WaitForSpans(TimeSpan.FromMinutes(2))) + throw new Exception("No span received in timeout"); + + _sender.Spans.Should().HaveCount(1); + var span = _sender.SpansOnFirstTransaction.First(); + + span.Name.Should().Be($"{ServiceBus.SegmentName} RECEIVE from {scope.QueueName}"); + span.Type.Should().Be(ApiConstants.TypeMessaging); + span.Subtype.Should().Be(ServiceBus.SubType); + } + [AzureCredentialsFact] public async Task Capture_Transaction_When_Receive_From_Queue() { diff --git a/test/Elastic.Apm.Azure.ServiceBus.Tests/MicrosoftAzureServiceBusDiagnosticListenerTests.cs b/test/Elastic.Apm.Azure.ServiceBus.Tests/MicrosoftAzureServiceBusDiagnosticListenerTests.cs index 5e3f862bc..953f918fb 100644 --- a/test/Elastic.Apm.Azure.ServiceBus.Tests/MicrosoftAzureServiceBusDiagnosticListenerTests.cs +++ b/test/Elastic.Apm.Azure.ServiceBus.Tests/MicrosoftAzureServiceBusDiagnosticListenerTests.cs @@ -1,4 +1,5 @@ using System; +using System.Linq; using System.Text; using System.Threading.Tasks; using Azure.Messaging.ServiceBus.Administration; @@ -156,6 +157,32 @@ await sender.ScheduleMessageAsync( destination.Service.Type.Should().Be(ApiConstants.TypeMessaging); } + [AzureCredentialsFact] + public async Task Capture_Span_When_Receive_From_Queue_Inside_Transaction() + { + await using var scope = await QueueScope.CreateWithQueue(_adminClient); + var sender = new MessageSender(_environment.ServiceBusConnectionString, scope.QueueName); + var receiver = new MessageReceiver(_environment.ServiceBusConnectionString, scope.QueueName, ReceiveMode.PeekLock); + + await sender.SendAsync( + new Message(Encoding.UTF8.GetBytes("test message"))).ConfigureAwait(false); + + await _agent.Tracer.CaptureTransaction("Receive messages", ApiConstants.TypeMessaging, async t => + { + await receiver.ReceiveAsync(TimeSpan.FromSeconds(30)).ConfigureAwait(false); + }); + + if (!_sender.WaitForSpans(TimeSpan.FromMinutes(2))) + throw new Exception("No span received in timeout"); + + _sender.Spans.Should().HaveCount(1); + var span = _sender.SpansOnFirstTransaction.First(); + + span.Name.Should().Be($"{ServiceBus.SegmentName} RECEIVE from {scope.QueueName}"); + span.Type.Should().Be(ApiConstants.TypeMessaging); + span.Subtype.Should().Be(ServiceBus.SubType); + } + [AzureCredentialsFact] public async Task Capture_Transaction_When_Receive_From_Queue() { From 3f9a734fc0e673d7cb6e0f0106bf284185005fe2 Mon Sep 17 00:00:00 2001 From: Russ Cam Date: Thu, 27 May 2021 11:21:31 +1000 Subject: [PATCH 06/21] Use TraceLogger as default logger in ASP.NET Full Framework (#1288) This commit adds a logger implementation, TraceLogger, that writes agent logs to a TraceSource with name "Elastic.Apm". This logger is configured as the default logger for ASP.NET Full Framework applications, which can use configuration to write log messages to file, debug window, Windows event log, etc. Add a section to docs with an example of how to configure the logger in web.config. Move the default log level from ConsoleLogger into DefaultValues. Closes #1263 --- docs/troubleshooting.asciidoc | 79 +++++++++++--- .../App_Start/LoggingConfig.cs | 3 +- .../Global.asax.cs | 20 ++-- .../ElasticApmModule.cs | 17 ++- .../Config/AbstractConfigurationReader.cs | 6 +- src/Elastic.Apm/Config/ConfigConsts.cs | 2 + src/Elastic.Apm/Logging/ConsoleLogger.cs | 7 +- src/Elastic.Apm/Logging/TraceLogger.cs | 101 ++++++++++++++++++ ...tCurrentExecutionSegmentsContainerTests.cs | 6 +- .../TestingConfig.cs | 7 +- 10 files changed, 207 insertions(+), 41 deletions(-) create mode 100644 src/Elastic.Apm/Logging/TraceLogger.cs diff --git a/docs/troubleshooting.asciidoc b/docs/troubleshooting.asciidoc index 5fcb5e031..d55cef742 100644 --- a/docs/troubleshooting.asciidoc +++ b/docs/troubleshooting.asciidoc @@ -46,15 +46,55 @@ This means the Agent will pick up the configured logging provider and log as any [[collect-logs-classic]] ==== ASP.NET Classic -Unlike ASP.NET Core, ASP.NET (classic) does not have a predefined logging system. -However, if you have a logging system in place, like NLog, Serilog, or similar, you can direct the agent logs into your -logging system by creating a bridge between the agent's internal logger and your logging system. +ASP.NET (classic) does not have a predefined logging system. By default, the agent is configured to +emit log messages to a +https://docs.microsoft.com/en-us/dotnet/api/system.diagnostics.tracesource[`System.Diagnostics.TraceSource`] +with the source name `"Elastic.Apm"`. The TraceSource adheres to the log levels defined in the +APM agent configuration. + +[IMPORTANT] +-- +System.Diagnostics.TraceSource requires the https://docs.microsoft.com/en-us/dotnet/framework/debug-trace-profile/how-to-compile-conditionally-with-trace-and-debug[`TRACE` compiler directive to be specified], which is specified +by default for both Debug and Release build configurations. +-- + +https://docs.microsoft.com/en-us/dotnet/api/system.diagnostics.tracelistener[TraceListeners] +can be configured to monitor log messages for the trace source, using the https://docs.microsoft.com/en-us/dotnet/framework/configure-apps/file-schema/trace-debug/system-diagnostics-element[``] section of +web.config. For example, the following web.config section writes Elastic.Apm log messages to a file +named my_log_file.log: + +[source,xml] +---- + + + + + <1> + + + + + + + +---- +<1> Define listeners under a source with name `"Elastic.Apm"` to capture agent logs + +[float] +[[collect-logs-class-other-logging-systems]] +===== Other logging systems + +If you have a logging system in place such as https://nlog-project.org/[NLog], https://serilog.net/[Serilog], +or similar, you can direct the agent logs into your logging system by creating an adapter between +the agent's internal logger and your logging system. First implement the `IApmLogger` interface from the `Elastic.Apm.Logging` namespace: [source,csharp] ---- -internal class ApmLoggerBridge : IApmLogger +internal class ApmLoggerAdapter : IApmLogger { private readonly Lazy _logger; public bool IsEnabled(ApmLogLevel level) @@ -71,31 +111,44 @@ internal class ApmLoggerBridge : IApmLogger } ---- -An example implementation for NLog can be seen https://github.com/elastic/apm-agent-dotnet/blob/master/sample/AspNetFullFrameworkSampleApp/App_Start/ApmLoggerToNLog.cs[in our GitHub repository]. - -Then tell the agent to use the `ApmLoggerBridge`. +An example implementation for NLog can be seen https://github.com/elastic/apm-agent-dotnet/blob/f6a33a185675b7b918af59d3333d94b32329a84a/sample/AspNetFullFrameworkSampleApp/App_Start/ApmLoggerToNLog.cs[in our GitHub repository]. -For this in ASP.NET (classic) you need to place the following code into the `Application_Start` method in the `HttpApplication` implementation of your app which is typically in the `Global.asx.cs` file: +Then tell the agent to use the `ApmLoggerAdapter`. For ASP.NET (classic), place the following code into the `Application_Start` +method in the `HttpApplication` implementation of your app which is typically in the `Global.asax.cs` file: [source,csharp] ---- -AgentDependencies.Logger = new ApmLoggerBridge(); +using Elastic.Apm.AspNetFullFramework; + +namespace MyApp +{ + public class MyApplication : HttpApplication + { + protected void Application_Start() + { + AgentDependencies.Logger = new ApmLoggerAdapter(); + + // other application setup... + } + } +} ---- -The `AgentDependencies` class lives in the `Elastic.Apm.AspNetFullFramework` namespace. -During initialization, the agent checks if an additional logger was configured--the agent only does this once, so it's important to set it as early in the process as possible (typically in the `Application_Start` method). +During initialization, the agent checks if an additional logger was configured-- the agent only does this once, so it's important +to set it as early in the process as possible, typically in the `Application_Start` method. [float] [[collect-logs-general]] ==== General .NET applications -If none of the above cases apply to your application, you can still use a bridge and redirect agent logs into a .NET logging system (like NLog, Serilog, or similar). +If none of the above cases apply to your application, you can still use a logger adapter and redirect agent logs into a .NET +logging system like NLog, Serilog, or similar. For this you'll need an `IApmLogger` implementation (see above) which you need to pass to the `Setup` method during agent setup: [source,csharp] ---- -Agent.Setup(new AgentComponents(logger: new ApmLoggerBridge())); +Agent.Setup(new AgentComponents(logger: new ApmLoggerAdapter())); ---- [float] diff --git a/sample/AspNetFullFrameworkSampleApp/App_Start/LoggingConfig.cs b/sample/AspNetFullFrameworkSampleApp/App_Start/LoggingConfig.cs index bd3f788ef..6b2ec90af 100644 --- a/sample/AspNetFullFrameworkSampleApp/App_Start/LoggingConfig.cs +++ b/sample/AspNetFullFrameworkSampleApp/App_Start/LoggingConfig.cs @@ -26,8 +26,6 @@ public class LoggingConfig public static void SetupLogging() { - var logFileEnvVarValue = Environment.GetEnvironmentVariable(LogFileEnvVarName); - var config = new LoggingConfiguration(); const string layout = "${date:format=yyyy-MM-dd HH\\:mm\\:ss.fff zzz}" + " | ${level:uppercase=true:padding=-5}" + // negative values cause right padding @@ -41,6 +39,7 @@ public static void SetupLogging() new PrefixingTraceTarget($"Elastic APM .NET {nameof(AspNetFullFrameworkSampleApp)}> "), LogMemoryTarget, new ConsoleTarget() }; + var logFileEnvVarValue = Environment.GetEnvironmentVariable(LogFileEnvVarName); if (logFileEnvVarValue != null) logTargets.Add(new FileTarget { FileName = logFileEnvVarValue, DeleteOldFileOnStartup = true }); foreach (var logTarget in logTargets) logTarget.Layout = layout; diff --git a/sample/AspNetFullFrameworkSampleApp/Global.asax.cs b/sample/AspNetFullFrameworkSampleApp/Global.asax.cs index 9d1f2b182..56b3a7473 100644 --- a/sample/AspNetFullFrameworkSampleApp/Global.asax.cs +++ b/sample/AspNetFullFrameworkSampleApp/Global.asax.cs @@ -14,21 +14,21 @@ using System.Web.Routing; using AspNetFullFrameworkSampleApp.Mvc; using Elastic.Apm; -using NLog; - +using NLog; + namespace AspNetFullFrameworkSampleApp { public class MvcApplication : HttpApplication { protected void Application_Start() - { - LoggingConfig.SetupLogging(); - - var logger = LogManager.GetCurrentClassLogger(); - logger.Info("Current process ID: {ProcessID}, ELASTIC_APM_SERVER_URLS: {ELASTIC_APM_SERVER_URLS}", - Process.GetCurrentProcess().Id, Environment.GetEnvironmentVariable("ELASTIC_APM_SERVER_URLS")); - - // Web API setup + { + LoggingConfig.SetupLogging(); + + var logger = LogManager.GetCurrentClassLogger(); + logger.Info("Current process ID: {ProcessID}, ELASTIC_APM_SERVER_URLS: {ELASTIC_APM_SERVER_URLS}", + Process.GetCurrentProcess().Id, Environment.GetEnvironmentVariable("ELASTIC_APM_SERVER_URLS")); + + // Web API setup HttpBatchHandler batchHandler = new DefaultHttpBatchHandler(GlobalConfiguration.DefaultServer) { ExecutionOrder = BatchExecutionOrder.NonSequential diff --git a/src/Elastic.Apm.AspNetFullFramework/ElasticApmModule.cs b/src/Elastic.Apm.AspNetFullFramework/ElasticApmModule.cs index 573b85bc7..7159e21dd 100644 --- a/src/Elastic.Apm.AspNetFullFramework/ElasticApmModule.cs +++ b/src/Elastic.Apm.AspNetFullFramework/ElasticApmModule.cs @@ -6,6 +6,7 @@ using System.Collections; using System.Collections.Generic; using System.Collections.Specialized; +using System.Configuration; using System.Linq; using System.Security.Claims; using System.Web; @@ -422,12 +423,22 @@ private static bool InitOnceForAllInstancesUnderLock(string dbgInstanceName) => Agent.Instance.Subscribe(new HttpDiagnosticsSubscriber()); }) ?? false; - private static IApmLogger BuildLogger() => AgentDependencies.Logger ?? ConsoleLogger.Instance; + private static IApmLogger CreateDefaultLogger() + { + var logLevel = ConfigurationManager.AppSettings[ConfigConsts.KeyNames.LogLevel]; + if (string.IsNullOrEmpty(logLevel)) + logLevel = Environment.GetEnvironmentVariable(ConfigConsts.EnvVarNames.LogLevel); + + var level = ConfigConsts.DefaultValues.LogLevel; + if (!string.IsNullOrEmpty(logLevel)) + Enum.TryParse(logLevel, true, out level); + + return new TraceLogger(level); + } private static AgentComponents CreateAgentComponents(string dbgInstanceName) { - var rootLogger = BuildLogger(); - + var rootLogger = AgentDependencies.Logger ?? CreateDefaultLogger(); var reader = ConfigHelper.CreateReader(rootLogger) ?? new FullFrameworkConfigReader(rootLogger); var agentComponents = new FullFrameworkAgentComponents(rootLogger, reader); diff --git a/src/Elastic.Apm/Config/AbstractConfigurationReader.cs b/src/Elastic.Apm/Config/AbstractConfigurationReader.cs index 091372b41..4d3ac7739 100644 --- a/src/Elastic.Apm/Config/AbstractConfigurationReader.cs +++ b/src/Elastic.Apm/Config/AbstractConfigurationReader.cs @@ -213,15 +213,15 @@ protected LogLevel ParseLogLevel(ConfigurationKeyValue kv) if (TryParseLogLevel(kv?.Value, out var level)) return level; if (kv?.Value == null) - _logger?.Debug()?.Log("No log level provided. Defaulting to log level '{DefaultLogLevel}'", ConsoleLogger.DefaultLogLevel); + _logger?.Debug()?.Log("No log level provided. Defaulting to log level '{DefaultLogLevel}'", DefaultValues.LogLevel); else { _logger?.Error() ?.Log("Failed parsing log level from {Origin}: {Key}, value: {Value}. Defaulting to log level '{DefaultLogLevel}'", - kv.ReadFrom, kv.Key, kv.Value, ConsoleLogger.DefaultLogLevel); + kv.ReadFrom, kv.Key, kv.Value, DefaultValues.LogLevel); } - return ConsoleLogger.DefaultLogLevel; + return DefaultValues.LogLevel; } protected Uri ParseServerUrl(ConfigurationKeyValue kv) => diff --git a/src/Elastic.Apm/Config/ConfigConsts.cs b/src/Elastic.Apm/Config/ConfigConsts.cs index 438a18a9a..8dc19dc68 100644 --- a/src/Elastic.Apm/Config/ConfigConsts.cs +++ b/src/Elastic.Apm/Config/ConfigConsts.cs @@ -6,6 +6,7 @@ using System.Collections.Generic; using Elastic.Apm.Cloud; using Elastic.Apm.Helpers; +using Elastic.Apm.Logging; namespace Elastic.Apm.Config { @@ -25,6 +26,7 @@ public static class DefaultValues public const bool CentralConfig = true; public const string CloudProvider = SupportedValues.CloudProviderAuto; public const int FlushIntervalInMilliseconds = 10_000; // 10 seconds + public const LogLevel LogLevel = Logging.LogLevel.Error; public const int MaxBatchEventCount = 10; public const int MaxQueueEventCount = 1000; public const string MetricsInterval = "30s"; diff --git a/src/Elastic.Apm/Logging/ConsoleLogger.cs b/src/Elastic.Apm/Logging/ConsoleLogger.cs index 1cb7ab7de..b214fefe8 100644 --- a/src/Elastic.Apm/Logging/ConsoleLogger.cs +++ b/src/Elastic.Apm/Logging/ConsoleLogger.cs @@ -4,13 +4,14 @@ using System; using System.IO; +using Elastic.Apm.Config; +using static Elastic.Apm.Config.ConfigConsts; namespace Elastic.Apm.Logging { internal class ConsoleLogger : IApmLogger, ILogLevelSwitchable { private static readonly object SyncRoot = new object(); - internal static readonly LogLevel DefaultLogLevel = LogLevel.Error; private readonly TextWriter _errorOut; private readonly TextWriter _standardOut; @@ -22,7 +23,7 @@ public ConsoleLogger(LogLevel level, TextWriter standardOut = null, TextWriter e _errorOut = errorOut ?? Console.Error; } - public static ConsoleLogger Instance { get; } = new ConsoleLogger(DefaultLogLevel); + public static ConsoleLogger Instance { get; } = new ConsoleLogger(DefaultValues.LogLevel); public LogLevelSwitch LogLevelSwitch { get; } @@ -30,7 +31,7 @@ public ConsoleLogger(LogLevel level, TextWriter standardOut = null, TextWriter e public static ConsoleLogger LoggerOrDefault(LogLevel? level) { - if (level.HasValue && level.Value != DefaultLogLevel) + if (level.HasValue && level.Value != DefaultValues.LogLevel) return new ConsoleLogger(level.Value); return Instance; diff --git a/src/Elastic.Apm/Logging/TraceLogger.cs b/src/Elastic.Apm/Logging/TraceLogger.cs new file mode 100644 index 000000000..d867f6e08 --- /dev/null +++ b/src/Elastic.Apm/Logging/TraceLogger.cs @@ -0,0 +1,101 @@ +// Licensed to Elasticsearch B.V under +// one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System; +using System.Diagnostics; +using System.Runtime.CompilerServices; +using System.Text; +using Elastic.Apm.Helpers; + +namespace Elastic.Apm.Logging +{ + /// + /// A logging implementation that logs to a with the source name Elastic.Apm + /// + internal class TraceLogger : IApmLogger, ILogLevelSwitchable + { + private const string SourceName = "Elastic.Apm"; + + private static readonly TraceSource TraceSource; + + static TraceLogger() => TraceSource = new TraceSource(SourceName); + + public TraceLogger(LogLevel level) => LogLevelSwitch = new LogLevelSwitch(level); + + public LogLevelSwitch LogLevelSwitch { get; } + + private LogLevel Level => LogLevelSwitch.Level; + + public bool IsEnabled(LogLevel level) => Level <= level; + + public void Log(LogLevel level, TState state, Exception e, Func formatter) + { + if (!IsEnabled(level)) return; + + var message = formatter(state, e); + var logLevel = LevelToString(level); + + StringBuilder builder; + string exceptionType = null; + var capacity = 51 + message.Length + logLevel.Length; + + if (e is null) + builder = new StringBuilder(capacity); + else + { + exceptionType = e.GetType().FullName; + builder = new StringBuilder(capacity + exceptionType.Length + e.Message.Length + e.StackTrace.Length); + } + + builder.Append('[') + .Append(DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.fff zzz")) + .Append("][") + .Append(logLevel) + .Append("] - ") + .Append(message); + + if (e != null) + { + builder.Append("+-> Exception: ") + .Append(exceptionType) + .Append(": ") + .AppendLine(e.Message) + .AppendLine(e.StackTrace); + } + + var logMessage = builder.ToString(); + for (var i = 0; i < TraceSource.Listeners.Count; i++) + { + var listener = TraceSource.Listeners[i]; + if (!listener.IsThreadSafe) + { + lock (listener) + listener.WriteLine(logMessage); + } + else + listener.WriteLine(logMessage); + } + + TraceSource.Flush(); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static string LevelToString(LogLevel level) + { + switch (level) + { + case LogLevel.Error: return "Error"; + case LogLevel.Warning: return "Warning"; + case LogLevel.Information: return "Info"; + case LogLevel.Debug: return "Debug"; + case LogLevel.Trace: return "Trace"; + case LogLevel.Critical: return "Critical"; + // ReSharper disable once RedundantCaseLabel + case LogLevel.None: + default: return "None"; + } + } + } +} diff --git a/test/Elastic.Apm.AspNetFullFramework.Tests/HttpContextCurrentExecutionSegmentsContainerTests.cs b/test/Elastic.Apm.AspNetFullFramework.Tests/HttpContextCurrentExecutionSegmentsContainerTests.cs index e7db33d55..6120feab4 100644 --- a/test/Elastic.Apm.AspNetFullFramework.Tests/HttpContextCurrentExecutionSegmentsContainerTests.cs +++ b/test/Elastic.Apm.AspNetFullFramework.Tests/HttpContextCurrentExecutionSegmentsContainerTests.cs @@ -38,9 +38,8 @@ public async Task Transaction_And_Spans_Captured_When_Large_Request() bytes.Should().BeGreaterThan(20_000); var content = new StringContent(json, Encoding.UTF8, "application/json"); - var client = new HttpClient(); var bulkSamplesUri = Consts.SampleApp.CreateUrl("/Database/Bulk"); - var response = await client.PostAsync(bulkSamplesUri, content).ConfigureAwait(false); + var response = await HttpClient.PostAsync(bulkSamplesUri, content).ConfigureAwait(false); var responseContent = await response.Content.ReadAsStringAsync(); response.IsSuccessStatusCode.Should().BeTrue(responseContent); @@ -62,9 +61,8 @@ public async Task Transaction_And_Spans_Captured_When_Controller_Action_Makes_As var count = 100; var content = new StringContent($"{{\"count\":{count}}}", Encoding.UTF8, "application/json"); - var client = new HttpClient(); var bulkSamplesUri = Consts.SampleApp.CreateUrl("/Database/Generate"); - var response = await client.PostAsync(bulkSamplesUri, content).ConfigureAwait(false); + var response = await HttpClient.PostAsync(bulkSamplesUri, content).ConfigureAwait(false); var responseContent = await response.Content.ReadAsStringAsync(); response.IsSuccessStatusCode.Should().BeTrue(responseContent); diff --git a/test/Elastic.Apm.Tests.Utilities/TestingConfig.cs b/test/Elastic.Apm.Tests.Utilities/TestingConfig.cs index 091ace83a..931546e18 100644 --- a/test/Elastic.Apm.Tests.Utilities/TestingConfig.cs +++ b/test/Elastic.Apm.Tests.Utilities/TestingConfig.cs @@ -10,6 +10,7 @@ using Elastic.Apm.Helpers; using Elastic.Apm.Logging; using Xunit.Abstractions; +using static Elastic.Apm.Config.ConfigConsts; namespace Elastic.Apm.Tests.Utilities { @@ -23,10 +24,10 @@ internal static class Options private const string SharedPrefix = "Elastic APM .NET Tests> {0}> "; internal static LogLevelOptionMetadata LogLevel = new LogLevelOptionMetadata( - "ELASTIC_APM_TESTS_LOG_LEVEL", ConsoleLogger.DefaultLogLevel, x => x.LogLevel); + "ELASTIC_APM_TESTS_LOG_LEVEL", DefaultValues.LogLevel, x => x.LogLevel); internal static LogLevelOptionMetadata LogLevelForTestingConfigParsing = new LogLevelOptionMetadata( - "ELASTIC_APM_TESTS_LOG_LEVEL_FOR_TESTING_CONFIG_PARSING", ConsoleLogger.DefaultLogLevel, x => x.LogLevelForTestingConfigParsing); + "ELASTIC_APM_TESTS_LOG_LEVEL_FOR_TESTING_CONFIG_PARSING", DefaultValues.LogLevel, x => x.LogLevelForTestingConfigParsing); internal static BoolOptionMetadata LogToConsoleEnabled = new BoolOptionMetadata( "ELASTIC_APM_TESTS_LOG_CONSOLE_ENABLED", !IsRunningInIde, x => x.LogToConsoleEnabled); @@ -238,7 +239,7 @@ internal class MutableSnapshot : ISnapshot internal MutableSnapshot(IRawConfigSnapshot rawConfigSnapshot, ITestOutputHelper xUnitOutputHelper) { - var tempLogger = BuildXunitOutputLogger(ConsoleLogger.DefaultLogLevel); + var tempLogger = BuildXunitOutputLogger(DefaultValues.LogLevel); Options.LogLevelForTestingConfigParsing.ParseAndSetProperty(rawConfigSnapshot, this, tempLogger); var parsingLogger = BuildXunitOutputLogger(LogLevelForTestingConfigParsing); From dadf26be008e586fa81af7378de93a6e79cb7d49 Mon Sep 17 00:00:00 2001 From: Russ Cam Date: Thu, 27 May 2021 17:01:28 +1000 Subject: [PATCH 07/21] Skip running Elasticsearch docker test when docker not available (#1312) This commit updates the Elasticsearch integration test that runs against a docker instance to be skipped when docker is not on the machine e.g. Windows .NET Core tests in CI --- .../TestsBase.cs | 2 ++ .../ElasticsearchTests.cs | 3 ++- .../ProfilingSessionTests.cs | 1 + .../Docker}/DockerFactAttribute.cs | 12 ++++++++---- 4 files changed, 13 insertions(+), 5 deletions(-) rename test/{Elastic.Apm.StackExchange.Redis.Tests => Elastic.Apm.Tests.Utilities/Docker}/DockerFactAttribute.cs (66%) diff --git a/test/Elastic.Apm.AspNetFullFramework.Tests/TestsBase.cs b/test/Elastic.Apm.AspNetFullFramework.Tests/TestsBase.cs index 065cad948..099ca0ecf 100644 --- a/test/Elastic.Apm.AspNetFullFramework.Tests/TestsBase.cs +++ b/test/Elastic.Apm.AspNetFullFramework.Tests/TestsBase.cs @@ -601,7 +601,9 @@ private void FullFwAssertValid(Api.System system) system.Should().NotBeNull(); system.DetectedHostName.Should().Be(new SystemInfoHelper(LoggerBase).GetHostName()); +#pragma warning disable 618 system.HostName.Should().Be(AgentConfig.HostName ?? system.DetectedHostName); +#pragma warning restore 618 } private void FullFwAssertValid(ErrorDto error) diff --git a/test/Elastic.Apm.Elasticsearch.Tests/ElasticsearchTests.cs b/test/Elastic.Apm.Elasticsearch.Tests/ElasticsearchTests.cs index a3ac35755..beff5b24b 100644 --- a/test/Elastic.Apm.Elasticsearch.Tests/ElasticsearchTests.cs +++ b/test/Elastic.Apm.Elasticsearch.Tests/ElasticsearchTests.cs @@ -8,6 +8,7 @@ using Elastic.Apm.Api; using Elastic.Apm.DiagnosticSource; using Elastic.Apm.Tests.Utilities; +using Elastic.Apm.Tests.Utilities.Docker; using Elasticsearch.Net; using Elasticsearch.Net.VirtualizedCluster; using FluentAssertions; @@ -25,7 +26,7 @@ public ElasticsearchTests(ElasticsearchFixture fixture) _client = new ElasticLowLevelClient(settings); } - [Fact] + [DockerFact] public async Task Elasticsearch_Span_Does_Not_Have_Http_Child_Span() { var payloadSender = new MockPayloadSender(); diff --git a/test/Elastic.Apm.StackExchange.Redis.Tests/ProfilingSessionTests.cs b/test/Elastic.Apm.StackExchange.Redis.Tests/ProfilingSessionTests.cs index a505f49c6..13b51e9c2 100644 --- a/test/Elastic.Apm.StackExchange.Redis.Tests/ProfilingSessionTests.cs +++ b/test/Elastic.Apm.StackExchange.Redis.Tests/ProfilingSessionTests.cs @@ -11,6 +11,7 @@ using DotNet.Testcontainers.Containers.Modules.Databases; using Elastic.Apm.Api; using Elastic.Apm.Tests.Utilities; +using Elastic.Apm.Tests.Utilities.Docker; using StackExchange.Redis; using FluentAssertions; diff --git a/test/Elastic.Apm.StackExchange.Redis.Tests/DockerFactAttribute.cs b/test/Elastic.Apm.Tests.Utilities/Docker/DockerFactAttribute.cs similarity index 66% rename from test/Elastic.Apm.StackExchange.Redis.Tests/DockerFactAttribute.cs rename to test/Elastic.Apm.Tests.Utilities/Docker/DockerFactAttribute.cs index 211514e63..dc94fccf7 100644 --- a/test/Elastic.Apm.StackExchange.Redis.Tests/DockerFactAttribute.cs +++ b/test/Elastic.Apm.Tests.Utilities/Docker/DockerFactAttribute.cs @@ -7,25 +7,29 @@ using ProcNet; using Xunit; -namespace Elastic.Apm.StackExchange.Redis.Tests +namespace Elastic.Apm.Tests.Utilities.Docker { /// /// Test method that should be run only if docker exists on the host /// public class DockerFactAttribute : FactAttribute { - public DockerFactAttribute() + private static readonly string _skip; + + static DockerFactAttribute() { try { var result = Proc.Start(new StartArguments("docker", "--version")); if (result.ExitCode != 0) - Skip = "docker not installed"; + _skip = "docker not installed"; } catch (Exception) { - Skip = "could not get version of docker"; + _skip = "could not get version of docker"; } } + + public DockerFactAttribute() => Skip = _skip; } } From c1f1b0e04f498f7cded73f0c2ebaef2040a6161a Mon Sep 17 00:00:00 2001 From: Ivan Fernandez Calvo Date: Thu, 27 May 2021 10:09:46 +0200 Subject: [PATCH 08/21] fix: use .NET native SDK for build and test (#1301) * test: run on Ubuntu 20.40 with Docker 20.10.3 * test: run only afected stages * test: run only afected tests * test: use latest Docker to have also the other failure in the history * test: grab docker logs * test: grab docker logs always * test: add some logs * fix: replace spaces with tabs * fix: add semicolons * test: use native SDK * test: only run one test * fix: wrong path * fix: install .NET SDK 5 * test: run all tests * Update Jenkinsfile * chore: restore full pipeline * fix: revert unneeded changes * fix: syntax error --- Jenkinsfile | 80 ++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 57 insertions(+), 23 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index abdb09565..2c1f67e57 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -18,7 +18,7 @@ pipeline { SLACK_CHANNEL = '#apm-agent-dotnet' } options { - timeout(time: 2, unit: 'HOURS') + timeout(time: 4, unit: 'HOURS') buildDiscarder(logRotator(numToKeepStr: '20', artifactNumToKeepStr: '20', daysToKeepStr: '30')) timestamps() ansiColor('xterm') @@ -28,7 +28,7 @@ pipeline { quietPeriod(10) } triggers { - issueCommentTrigger('(?i).*(?:jenkins\\W+)?run\\W+(?:the\\W+)?(?:benchmark\\W+)?tests(?:\\W+please)?.*') + issueCommentTrigger('(?i)(.*(?:jenkins\\W+)?run\\W+(?:the\\W+)?(?:benchmark\\W+)?tests(?:\\W+please)?.*|/test)') } parameters { booleanParam(name: 'Run_As_Master_Branch', defaultValue: false, description: 'Allow to run any steps on a PR, some steps normally only run on master branch.') @@ -72,20 +72,20 @@ pipeline { Make sure there are no code style violation in the repo. */ stages{ - // Disable until https://github.com/elastic/apm-agent-dotnet/issues/563 - // stage('CodeStyleCheck') { - // steps { - // withGithubNotify(context: 'CodeStyle check') { - // deleteDir() - // unstash 'source' - // dir("${BASE_DIR}"){ - // dotnet(){ - // sh label: 'Install and run dotnet/format', script: '.ci/linux/codestyle.sh' - // } - // } - // } - // } - // } + // Disable until https://github.com/elastic/apm-agent-dotnet/issues/563 + // stage('CodeStyleCheck') { + // steps { + // withGithubNotify(context: 'CodeStyle check') { + // deleteDir() + // unstash 'source' + // dir("${BASE_DIR}"){ + // dotnet(){ + // sh label: 'Install and run dotnet/format', script: '.ci/linux/codestyle.sh' + // } + // } + // } + // } + // } /** Build the project from code.. */ @@ -124,9 +124,11 @@ pipeline { withGithubNotify(context: 'Test - Linux', tab: 'tests') { deleteDir() unstash 'source' - dir("${BASE_DIR}"){ - dotnet(){ - sh label: 'Test & coverage', script: '.ci/linux/test.sh' + filebeat(output: "docker.log"){ + dir("${BASE_DIR}"){ + dotnet(){ + sh label: 'Test & coverage', script: '.ci/linux/test.sh' + } } } } @@ -520,16 +522,48 @@ def cleanDir(path){ } def dotnet(Closure body){ - def dockerTagName = 'docker.elastic.co/observability-ci/apm-agent-dotnet-sdk-linux:latest' - sh label: 'Docker build', script: "docker build --tag ${dockerTagName} .ci/docker/sdk-linux" + def homePath = "${env.WORKSPACE}/${env.BASE_DIR}" - docker.image("${dockerTagName}").inside("-e HOME='${homePath}' -v /var/run/docker.sock:/var/run/docker.sock"){ + withEnv([ + "HOME=${homePath}", + "DOTNET_ROOT=${homePath}/.dotnet", + "PATH+DOTNET=${homePath}/.dotnet/tools:${homePath}/.dotnet" + ]){ + sh(label: 'Install dotnet SDK', script: """ + mkdir -p \${DOTNET_ROOT} + # Download .Net SDK installer script + curl -s -O -L https://dotnet.microsoft.com/download/dotnet-core/scripts/v1/dotnet-install.sh + chmod ugo+rx dotnet-install.sh + + # Install .Net SDKs + ./dotnet-install.sh --install-dir "\${DOTNET_ROOT}" -version '2.1.505' + ./dotnet-install.sh --install-dir "\${DOTNET_ROOT}" -version '3.0.103' + ./dotnet-install.sh --install-dir "\${DOTNET_ROOT}" -version '3.1.100' + ./dotnet-install.sh --install-dir "\${DOTNET_ROOT}" -version '5.0.203' + """) withAzureCredentials(path: "${homePath}", credentialsFile: '.credentials.json') { - body() + withTerraform(){ + body() + } } } } +def withTerraform(Closure body){ + def binDir = "${HOME}/bin" + withEnv([ + "PATH+TERRAFORM=${binDir}" + ]){ + sh(label:'Install Terraform', script: """ + mkdir -p ${binDir} + cd ${binDir} + curl -sSL -o terraform.zip https://releases.hashicorp.com/terraform/0.15.3/terraform_0.15.3_linux_amd64.zip + unzip terraform.zip + """) + body() + } +} + def release(Map args = [:]){ def secret = args.secret def withSuffix = args.get('withSuffix', false) From eb273377d5061c6b9fca58cea4f29f93ddcce79c Mon Sep 17 00:00:00 2001 From: Russ Cam Date: Fri, 28 May 2021 01:26:37 +1000 Subject: [PATCH 09/21] Use Logger to log exception in AgentComponents initialization (#1305) This commit fixes a bug in logging an exception that may occur during agent initialization; the Logger property is used instead of the logger argument, the former will always be initialized with a value. Fixes #1254 --- src/Elastic.Apm/AgentComponents.cs | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/Elastic.Apm/AgentComponents.cs b/src/Elastic.Apm/AgentComponents.cs index b353cd05d..4ae3c8276 100644 --- a/src/Elastic.Apm/AgentComponents.cs +++ b/src/Elastic.Apm/AgentComponents.cs @@ -53,22 +53,21 @@ IApmServerInfo apmServerInfo HttpTraceConfiguration = new HttpTraceConfiguration(); + TracerInternal = new Tracer(Logger, Service, PayloadSender, ConfigStore, + currentExecutionSegmentsContainer ?? new CurrentExecutionSegmentsContainer(), ApmServerInfo); + if (ConfigurationReader.Enabled) { CentralConfigFetcher = centralConfigFetcher ?? new CentralConfigFetcher(Logger, ConfigStore, Service); MetricsCollector = metricsCollector ?? new MetricsCollector(Logger, PayloadSender, ConfigStore); MetricsCollector.StartCollecting(); } - - TracerInternal = new Tracer(Logger, Service, PayloadSender, ConfigStore, - currentExecutionSegmentsContainer ?? new CurrentExecutionSegmentsContainer(), ApmServerInfo); - - if (!ConfigurationReader.Enabled) - Logger?.Info()?.Log("The Elastic APM .NET Agent is disabled - the agent won't capture traces and metrics."); + else + Logger.Info()?.Log("The Elastic APM .NET Agent is disabled - the agent won't capture traces and metrics."); } catch (Exception e) { - logger?.Error()?.LogException(e, "Failed initializing agent."); + Logger.Error()?.LogException(e, "Failed initializing agent."); } } From 8dd60c44bc93c7e1c1c774474311e29bd75080d6 Mon Sep 17 00:00:00 2001 From: Russ Cam Date: Fri, 28 May 2021 02:00:54 +1000 Subject: [PATCH 10/21] Capture errors with startup hook auto instrumentation (#1298) This commit updates the startup hook auto instrumentation feature to capture thrown exceptions through the subscribed AspNetCoreDiagnosticListener. When an unhandled exception is thrown, ASP.NET Core's DeveloperExceptionPageMiddleware will raise a diagnostic event with key "Microsoft.AspNetCore.Diagnostics.UnhandledException" if there is a listener listening to the event. If no listener is subscribed, the event is not captured. Update how assemblies are loaded in the startup hook auto instrumentation. Register an event handler to the AssemblyLoadContext.Default.Resolve event to load assemblies that exist in the loader directory with matching version and public key. For assemblies starting with Elastic.Apm, that may contain DiagnosticListeners, load into AssemblyLoadContext.Default to allow listeners to register via DiagnosticListener.AllListeners.Subscribe(). For any other assemblies, load them into a separate AssemblyLoadContext to avoid version conflicts with assemblies that the application may reference. An example of the problem this avoids is System.Reflection.Metadata; The APM agent's source included version of Ben.Demystifier depends on System.Reflection.Metadata 5.0.0, but an application may reference a different version, such as ASP.NET Core 3.0, which references System.Reflection.Metadata 1.4.4.0 by default. Add integration tests for capturing errors from startup hooks for netcoreapp3.0, netcoreapp3.1 and net5.0. Update documentation for startup hooks to indicate that this feature is supported in .NET Core 3.0 and newer. .NET Core 2.2 is End of Life (EOL) and no longer supported by Microsoft. In testing startup hooks with .NET Core 2.2, Diagnostic Listeners do not subscribe to ASP.NET Core diagnostic events. On initial investigation, the System.Diagnostics.DiagnosticSource assembly loaded in the startup hooks is version 4.0.3.1. When listeners come to subscribe, System.Diagnostics.DiagnosticSource 4.0.4.0 that listeners are compiled against is loaded into the separate AssemblyLoadContext, which might be why listeners do not end up subscribing. Considering .NET Core 2.2 is EOL however, no further effort has been put into making this work. Fixes #1233 --- docs/setup.asciidoc | 2 +- .../Controllers/HomeController.cs | 2 + .../AspNetCoreDiagnosticListener.cs | 2 +- .../AspNetCoreDiagnosticSubscriber.cs | 2 +- src/Elastic.Apm.StartupHook.Loader/Loader.cs | 18 +--- src/ElasticApmAgentStartupHook/StartupHook.cs | 92 ++++++++++++++----- .../StartupHookLogger.cs | 9 +- .../StartupHookTests.cs | 52 +++++++++++ 8 files changed, 135 insertions(+), 44 deletions(-) diff --git a/docs/setup.asciidoc b/docs/setup.asciidoc index da401442e..059e4fcb2 100644 --- a/docs/setup.asciidoc +++ b/docs/setup.asciidoc @@ -168,7 +168,7 @@ The following example only turns on outgoing HTTP monitoring (so, for instance, [[zero-code-change-setup]] ==== Zero code change setup on .NET Core (added[1.7]) -If you can't or don't want to reference NuGet packages in your application, you can use the startup hook feature to inject the agent during startup, if your application runs on .NET Core. This feature is supported on .NET Core 2.2 and newer versions. +If you can't or don't want to reference NuGet packages in your application, you can use the startup hook feature to inject the agent during startup, if your application runs on .NET Core. This feature is supported on .NET Core 3.0 and newer versions. Steps: diff --git a/sample/Elastic.Apm.StartupHook.Sample/Controllers/HomeController.cs b/sample/Elastic.Apm.StartupHook.Sample/Controllers/HomeController.cs index 2dce775c3..454e487fc 100644 --- a/sample/Elastic.Apm.StartupHook.Sample/Controllers/HomeController.cs +++ b/sample/Elastic.Apm.StartupHook.Sample/Controllers/HomeController.cs @@ -21,5 +21,7 @@ public class HomeController : Controller [ResponseCache(Duration = 0, Location = ResponseCacheLocation.None, NoStore = true)] public IActionResult Error() => View(new ErrorViewModel { RequestId = Activity.Current?.Id ?? HttpContext.TraceIdentifier }); + + public IActionResult Exception() => throw new Exception("Exception thrown from controller action"); } } diff --git a/src/Elastic.Apm.AspNetCore/DiagnosticListener/AspNetCoreDiagnosticListener.cs b/src/Elastic.Apm.AspNetCore/DiagnosticListener/AspNetCoreDiagnosticListener.cs index 2f745e029..ed3c1336b 100644 --- a/src/Elastic.Apm.AspNetCore/DiagnosticListener/AspNetCoreDiagnosticListener.cs +++ b/src/Elastic.Apm.AspNetCore/DiagnosticListener/AspNetCoreDiagnosticListener.cs @@ -66,7 +66,7 @@ protected override void HandleOnNext(KeyValuePair kv) } } break; - case "Microsoft.AspNetCore.Diagnostics.UnhandledException": //Called when exception handler is registrered + case "Microsoft.AspNetCore.Diagnostics.UnhandledException": //Called when exception handler is registered case "Microsoft.AspNetCore.Diagnostics.HandledException": if (!(_defaultHttpContextFetcher.Fetch(kv.Value) is DefaultHttpContext httpContextDiagnosticsUnhandledException)) return; if (!(_exceptionContextPropertyFetcher.Fetch(kv.Value) is Exception diagnosticsException)) return; diff --git a/src/Elastic.Apm.AspNetCore/DiagnosticListener/AspNetCoreDiagnosticSubscriber.cs b/src/Elastic.Apm.AspNetCore/DiagnosticListener/AspNetCoreDiagnosticSubscriber.cs index 1114fbf4b..6dec5830f 100644 --- a/src/Elastic.Apm.AspNetCore/DiagnosticListener/AspNetCoreDiagnosticSubscriber.cs +++ b/src/Elastic.Apm.AspNetCore/DiagnosticListener/AspNetCoreDiagnosticSubscriber.cs @@ -4,7 +4,7 @@ namespace Elastic.Apm.AspNetCore.DiagnosticListener { /// - /// A Diagnostic listner to create transactions based on diagnostic source events for ASP.NET Core. + /// A Diagnostic listener to create transactions based on diagnostic source events for ASP.NET Core. /// This itself manages all transaction and error capturing without the need for a middleware. /// public class AspNetCoreDiagnosticSubscriber : IDiagnosticsSubscriber diff --git a/src/Elastic.Apm.StartupHook.Loader/Loader.cs b/src/Elastic.Apm.StartupHook.Loader/Loader.cs index 6d11365dc..27b577d73 100644 --- a/src/Elastic.Apm.StartupHook.Loader/Loader.cs +++ b/src/Elastic.Apm.StartupHook.Loader/Loader.cs @@ -20,7 +20,7 @@ namespace Elastic.Apm.StartupHook.Loader { /// - /// Loads the agent assemblies, its dependent assemblies and starts it + /// Starts the agent /// internal class Loader { @@ -37,23 +37,9 @@ private static string AssemblyDirectory } /// - /// Initializes assemblies and starts the agent + /// Initializes and starts the agent /// public static void Initialize() - { - var agentLibsToLoad = new[]{ "Elastic.Apm", "Elastic.Apm.Extensions.Hosting", "Elastic.Apm.AspNetCore", "Elastic.Apm.EntityFrameworkCore", "Elastic.Apm.SqlClient", "Elastic.Apm.GrpcClient", "Elastic.Apm.Elasticsearch" }; - var agentDependencyLibsToLoad = new[] { "System.Diagnostics.PerformanceCounter", "Microsoft.Diagnostics.Tracing.TraceEvent", "Elasticsearch.Net" }; - - foreach (var libToLoad in agentDependencyLibsToLoad) - AssemblyLoadContext.Default.LoadFromAssemblyPath(Path.Combine(AssemblyDirectory, libToLoad + ".dll")); - foreach (var libToLoad in agentLibsToLoad) - AssemblyLoadContext.Default.LoadFromAssemblyPath(Path.Combine(AssemblyDirectory, libToLoad + ".dll")); - - StartAgent(); - } - - [MethodImpl(MethodImplOptions.NoInlining)] - private static void StartAgent() { Agent.Setup(new AgentComponents()); diff --git a/src/ElasticApmAgentStartupHook/StartupHook.cs b/src/ElasticApmAgentStartupHook/StartupHook.cs index 2c7d6446f..67c5a93a0 100644 --- a/src/ElasticApmAgentStartupHook/StartupHook.cs +++ b/src/ElasticApmAgentStartupHook/StartupHook.cs @@ -14,7 +14,9 @@ // ReSharper disable once CheckNamespace - per doc. this must be called StartupHook without a namespace with an Initialize method. internal class StartupHook { - private const string ElasticApmStartuphookLoaderDll = "Elastic.Apm.StartupHook.Loader.dll"; + private const string LoaderDll = "Elastic.Apm.StartupHook.Loader.dll"; + private const string LoaderTypeName = "Elastic.Apm.StartupHook.Loader.Loader"; + private const string LoaderTypeMethod = "Initialize"; private const string SystemDiagnosticsDiagnosticsource = "System.Diagnostics.DiagnosticSource"; private static readonly byte[] SystemDiagnosticsDiagnosticSourcePublicKeyToken = { 204, 123, 19, 255, 205, 45, 221, 81 }; @@ -40,7 +42,7 @@ public static void Initialize() var assemblies = AppDomain.CurrentDomain.GetAssemblies(); - _logger.WriteLine($"Assemblies loaded:{Environment.NewLine}{string.Join(",", assemblies.Select(a => a.GetName()))}"); + _logger.WriteLine($"Assemblies loaded:{Environment.NewLine}{string.Join(Environment.NewLine, assemblies.Select(a => a.GetName()))}"); var diagnosticSourceAssemblies = assemblies .Where(a => a.GetName().Name.Equals(SystemDiagnosticsDiagnosticsource, StringComparison.Ordinal)) @@ -63,7 +65,9 @@ public static void Initialize() break; } - Assembly loader = null; + Assembly loaderAssembly = null; + string loaderDirectory; + if (diagnosticSourceAssembly is null) { // use agent compiled against the highest version of System.Diagnostics.DiagnosticSource @@ -72,9 +76,9 @@ public static void Initialize() .OrderByDescending(d => VersionRegex.Match(d).Groups["major"].Value) .First(); - var versionDirectory = Path.Combine(startupHookDirectory, highestAvailableAgent); - loader = AssemblyLoadContext.Default - .LoadFromAssemblyPath(Path.Combine(versionDirectory, ElasticApmStartuphookLoaderDll)); + loaderDirectory = Path.Combine(startupHookDirectory, highestAvailableAgent); + loaderAssembly = AssemblyLoadContext.Default + .LoadFromAssemblyPath(Path.Combine(loaderDirectory, LoaderDll)); } else { @@ -91,11 +95,11 @@ public static void Initialize() var diagnosticSourceVersion = diagnosticSourceAssemblyName.Version; _logger.WriteLine($"{SystemDiagnosticsDiagnosticsource} {diagnosticSourceVersion} loaded"); - var versionDirectory = Path.Combine(startupHookDirectory, $"{diagnosticSourceVersion.Major}.0.0"); - if (Directory.Exists(versionDirectory)) + loaderDirectory = Path.Combine(startupHookDirectory, $"{diagnosticSourceVersion.Major}.0.0"); + if (Directory.Exists(loaderDirectory)) { - loader = AssemblyLoadContext.Default - .LoadFromAssemblyPath(Path.Combine(versionDirectory, ElasticApmStartuphookLoaderDll)); + loaderAssembly = AssemblyLoadContext.Default + .LoadFromAssemblyPath(Path.Combine(loaderDirectory, LoaderDll)); } else { @@ -104,7 +108,53 @@ public static void Initialize() } } - InvokerLoaderMethod(loader); + if (loaderAssembly is null) + { + _logger.WriteLine( + $"No {LoaderDll} assembly loaded. Agent not loaded"); + } + + LoadAssembliesFromLoaderDirectory(loaderDirectory); + InvokerLoaderMethod(loaderAssembly); + } + + /// + /// Loads assemblies from the loader directory if they exist + /// + /// + private static void LoadAssembliesFromLoaderDirectory(string loaderDirectory) + { + var context = new ElasticApmAssemblyLoadContext(); + AssemblyLoadContext.Default.Resolving += (_, name) => + { + var assemblyPath = Path.Combine(loaderDirectory, name.Name + ".dll"); + if (File.Exists(assemblyPath)) + { + try + { + var assemblyName = AssemblyName.GetAssemblyName(assemblyPath); + if (name.Version == assemblyName.Version) + { + var keyToken = name.GetPublicKeyToken(); + var assemblyKeyToken = assemblyName.GetPublicKeyToken(); + if (keyToken.SequenceEqual(assemblyKeyToken)) + { + // load Elastic.Apm assemblies with the default assembly load context, to allow DiagnosticListeners to subscribe. + // For all other dependencies, load with a separate load context to not conflict with application dependencies. + return name.Name.StartsWith("Elastic.Apm", StringComparison.Ordinal) + ? AssemblyLoadContext.Default.LoadFromAssemblyPath(assemblyPath) + : context.LoadFromAssemblyPath(assemblyPath); + } + } + } + catch (Exception e) + { + _logger.WriteLine(e.ToString()); + } + } + + return null; + }; } /// @@ -125,31 +175,25 @@ private static string PublicKeyTokenBytesToString(byte[] publicKeyToken) /// The loader assembly private static void InvokerLoaderMethod(Assembly loaderAssembly) { - if (loaderAssembly is null) - return; - - const string loaderTypeName = "Elastic.Apm.StartupHook.Loader.Loader"; - const string loaderTypeMethod = "Initialize"; - - _logger.WriteLine($"Get {loaderTypeName} type"); - var loaderType = loaderAssembly.GetType(loaderTypeName); + _logger.WriteLine($"Get {LoaderTypeName} type"); + var loaderType = loaderAssembly.GetType(LoaderTypeName); if (loaderType is null) { - _logger.WriteLine($"{loaderTypeName} type is null"); + _logger.WriteLine($"{LoaderTypeName} type is null"); return; } - _logger.WriteLine($"Get {loaderTypeName}.{loaderTypeMethod} method"); - var initializeMethod = loaderType.GetMethod(loaderTypeMethod, BindingFlags.Public | BindingFlags.Static); + _logger.WriteLine($"Get {LoaderTypeName}.{LoaderTypeMethod} method"); + var initializeMethod = loaderType.GetMethod(LoaderTypeMethod, BindingFlags.Public | BindingFlags.Static); if (initializeMethod is null) { - _logger.WriteLine($"{loaderTypeName}.{loaderTypeMethod} method is null"); + _logger.WriteLine($"{LoaderTypeName}.{LoaderTypeMethod} method is null"); return; } - _logger.WriteLine($"Invoke {loaderTypeName}.{loaderTypeMethod} method"); + _logger.WriteLine($"Invoke {LoaderTypeName}.{LoaderTypeMethod} method"); initializeMethod.Invoke(null, null); } } diff --git a/src/ElasticApmAgentStartupHook/StartupHookLogger.cs b/src/ElasticApmAgentStartupHook/StartupHookLogger.cs index ac3b6f0bf..290b6e59d 100644 --- a/src/ElasticApmAgentStartupHook/StartupHookLogger.cs +++ b/src/ElasticApmAgentStartupHook/StartupHookLogger.cs @@ -5,13 +5,20 @@ using System; using System.IO; +using System.Reflection; +using System.Runtime.Loader; namespace ElasticApmStartupHook { + internal class ElasticApmAssemblyLoadContext : AssemblyLoadContext + { + protected override Assembly Load(AssemblyName assemblyName) => null; + } + /// /// Logs startup hook process, useful for debugging purposes. /// - public class StartupHookLogger + internal class StartupHookLogger { private readonly bool _enabled; private readonly string _logPath; diff --git a/test/Elastic.Apm.StartupHook.Tests/StartupHookTests.cs b/test/Elastic.Apm.StartupHook.Tests/StartupHookTests.cs index 4974af8af..d46ce0bae 100644 --- a/test/Elastic.Apm.StartupHook.Tests/StartupHookTests.cs +++ b/test/Elastic.Apm.StartupHook.Tests/StartupHookTests.cs @@ -70,6 +70,58 @@ public async Task Auto_Instrument_With_StartupHook_Should_Capture_Transaction(st await apmServer.StopAsync(); } + [Theory] + [InlineData("netcoreapp3.0")] + [InlineData("netcoreapp3.1")] + [InlineData("net5.0")] + public async Task Auto_Instrument_With_StartupHook_Should_Capture_Error(string targetFramework) + { + var apmLogger = new InMemoryBlockingLogger(LogLevel.Error); + var apmServer = new MockApmServer(apmLogger, nameof(Auto_Instrument_With_StartupHook_Should_Capture_Error)); + var port = apmServer.FindAvailablePortToListen(); + apmServer.RunInBackground(port); + var transactionWaitHandle = new ManualResetEvent(false); + var errorWaitHandle = new ManualResetEvent(false); + + apmServer.OnReceive += o => + { + if (o is TransactionDto) + transactionWaitHandle.Set(); + if (o is ErrorDto) + errorWaitHandle.Set(); + }; + + using (var sampleApp = new SampleApplication()) + { + var environmentVariables = new Dictionary + { + [EnvVarNames.ServerUrl] = $"http://localhost:{port}", + [EnvVarNames.CloudProvider] = "none" + }; + + var uri = sampleApp.Start(targetFramework, environmentVariables); + var builder = new UriBuilder(uri) { Path = "Home/Exception" }; + var client = new HttpClient(); + var response = await client.GetAsync(builder.Uri); + + response.IsSuccessStatusCode.Should().BeFalse(); + + transactionWaitHandle.WaitOne(TimeSpan.FromMinutes(2)); + apmServer.ReceivedData.Transactions.Should().HaveCount(1); + + var transaction = apmServer.ReceivedData.Transactions.First(); + transaction.Name.Should().Be("GET Home/Exception"); + + errorWaitHandle.WaitOne(TimeSpan.FromMinutes(2)); + apmServer.ReceivedData.Errors.Should().HaveCount(1); + + var error = apmServer.ReceivedData.Errors.First(); + error.Culprit.Should().Be("Elastic.Apm.StartupHook.Sample.Controllers.HomeController"); + } + + await apmServer.StopAsync(); + } + [Theory] [InlineData("netcoreapp3.0", ".NET Core", "3.0.0.0")] [InlineData("netcoreapp3.1", ".NET Core", "3.1.0.0")] From e0452d6c3927c2e37e44188cacb59f1261b6026f Mon Sep 17 00:00:00 2001 From: Gergely Kalapos Date: Thu, 27 May 2021 18:22:20 +0200 Subject: [PATCH 11/21] Fix nullref in Elastic.Apm.Extensions.Logging (#1311) * Fix nullref in Elastic.Apm.Extensions.Logging * Update StacktraceHelper.cs --- .../ApmErrorLogger.cs | 2 +- src/Elastic.Apm/Helpers/StacktraceHelper.cs | 6 +-- .../CaptureApmErrorsTests.cs | 46 +++++++++++++++---- 3 files changed, 40 insertions(+), 14 deletions(-) diff --git a/src/Elastic.Apm.Extensions.Logging/ApmErrorLogger.cs b/src/Elastic.Apm.Extensions.Logging/ApmErrorLogger.cs index c9b8779dc..6a234967f 100644 --- a/src/Elastic.Apm.Extensions.Logging/ApmErrorLogger.cs +++ b/src/Elastic.Apm.Extensions.Logging/ApmErrorLogger.cs @@ -42,7 +42,7 @@ public void Log(LogLevel logLevel, EventId eventId, TState state, Except if (_agent is ApmAgent apmAgent && exception != null) { - errorLog.StackTrace = StacktraceHelper.GenerateApmStackTrace(exception, null, "CaptureErrorLogsAsApmError", + errorLog.StackTrace = StacktraceHelper.GenerateApmStackTrace(exception, _agent.Logger, "CaptureErrorLogsAsApmError", apmAgent.ConfigurationReader, apmAgent.Components.ApmServerInfo); } diff --git a/src/Elastic.Apm/Helpers/StacktraceHelper.cs b/src/Elastic.Apm/Helpers/StacktraceHelper.cs index ab8998744..2ce614929 100644 --- a/src/Elastic.Apm/Helpers/StacktraceHelper.cs +++ b/src/Elastic.Apm/Helpers/StacktraceHelper.cs @@ -98,7 +98,7 @@ internal static List GenerateApmStackTrace(StackFrame[] fram } catch (Exception e) { - logger?.Warning()?.LogException(e, "Failed capturing stacktrace for {ApmContext}", dbgCapturingFor); + logger.Warning()?.LogException(e, "Failed capturing stacktrace for {ApmContext}", dbgCapturingFor); } return retVal; @@ -135,7 +135,7 @@ internal static List GenerateApmStackTrace(Exception excepti } catch (Exception e) { - logger?.Debug() + logger.Debug() ? .LogException(e, "Failed generating stack trace with EnhancedStackTrace - using fallback without demystification"); // Fallback, see https://github.com/elastic/apm-agent-dotnet/issues/957 @@ -146,7 +146,7 @@ internal static List GenerateApmStackTrace(Exception excepti } catch (Exception e) { - logger?.Warning() + logger.Warning() ?.Log("Failed extracting stack trace from exception for {ApmContext}." + " Exception for failure to extract: {ExceptionForFailureToExtract}." + " Exception to extract from: {ExceptionToExtractFrom}.", diff --git a/test/Elastic.Apm.Extensions.Logging.Tests/CaptureApmErrorsTests.cs b/test/Elastic.Apm.Extensions.Logging.Tests/CaptureApmErrorsTests.cs index ef61a03e5..71ebb9393 100644 --- a/test/Elastic.Apm.Extensions.Logging.Tests/CaptureApmErrorsTests.cs +++ b/test/Elastic.Apm.Extensions.Logging.Tests/CaptureApmErrorsTests.cs @@ -1,4 +1,11 @@ -using System.Threading.Tasks; +// Licensed to Elasticsearch B.V under +// one or more agreements. +// Elasticsearch B.V licenses this file to you under the Apache 2.0 License. +// See the LICENSE file in the project root for more information + +using System; +using System.Linq; +using System.Threading.Tasks; using Elastic.Apm.Extensions.Hosting; using Elastic.Apm.Report; using Elastic.Apm.Tests.Utilities; @@ -27,22 +34,41 @@ public async Task CaptureErrorLogsAsApmError() payloadSender.FirstError.Log.Message.Should().Be("This is a sample error log message, with a sample value: 42"); payloadSender.FirstError.Log.ParamMessage.Should().Be("This is a sample error log message, with a sample value: {intParam}"); + // Test a log with exception + var logger = (ILogger)hostBuilder.Services.GetService(typeof(ILogger)); + + try + { + throw new Exception(); + } + catch (Exception e) + { + logger.LogError(e, "error log with exception"); + } + + payloadSender.WaitForErrors(); + payloadSender.Errors.Should().NotBeEmpty(); + payloadSender.Errors.Where(n => n.Log.Message == "error log with exception" && + n.Log.StackTrace != null && n.Log.StackTrace.Count > 0) + .Should() + .NotBeNullOrEmpty(); + await hostBuilder.StopAsync(); } private static IHostBuilder CreateHostBuilder(MockPayloadSender payloadSender = null) => - Host.CreateDefaultBuilder() - .ConfigureServices(n => n.AddSingleton(serviceProvider => payloadSender)) - .ConfigureServices((context, services) => { services.AddHostedService(); }) - .ConfigureLogging((hostingContext, logging) => - { - logging.ClearProviders(); + Host.CreateDefaultBuilder() + .ConfigureServices(n => n.AddSingleton(_ => payloadSender)) + .ConfigureServices((_, services) => { services.AddHostedService(); }) + .ConfigureLogging((_, logging) => + { + logging.ClearProviders(); #if NET5_0 - logging.AddSimpleConsole(o => o.IncludeScopes = true); + logging.AddSimpleConsole(o => o.IncludeScopes = true); #else logging.AddConsole(options => options.IncludeScopes = true); #endif - }) - .UseElasticApm(); + }) + .UseElasticApm(); } } From ecfe0d02ee33d22706236b4cb31126d15a02507f Mon Sep 17 00:00:00 2001 From: Gergely Kalapos Date: Thu, 27 May 2021 19:44:13 +0200 Subject: [PATCH 12/21] Prepare release v.1.10.0 (#1314) * Prepare release v.1.10.0 * Update CHANGELOG.asciidoc * Update Directory.Build.props --- CHANGELOG.asciidoc | 25 +++++++++++++++++++++++++ src/Directory.Build.props | 10 +++++----- 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index d6674a229..2c0ebed9b 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -23,6 +23,31 @@ endif::[] [[release-notes-1.x]] === .NET Agent version 1.x +[[release-notes-1.10.0]] +==== 1.10.0 + +[float] +===== Features +- {pull}1225[#1225] Add instrumentation for Azure Service Bus (issue: {issue}1157[#1157]) +- {pull}1247[#1247] Add Azure storage integration (issues: {issue}1156[#1156] and {issue}1155[#1155]) +- {pull}1241[#1241] Internalize `Newtonsoft.Json` - no more dependency on `Newtonsoft.Json` +- {pull}1275[#1275] Internalize `Ben.Demystifier` - no more dependency on `Ben.Demystifier` (issue: {issue}1232[#1232]) +- {pull}1215[#1215] Add MongoDb support (issue: {issue}1158[#1158]) +- {pull}1277[#1277] Capture inner exceptions (issue: {issue}1267[#1267]) +- {pull}1290[#1290] Add configured hostname (issue: {issue}1289[#1289]) +- {pull}1288[#1288] Use TraceLogger as default logger in ASP.NET Full Framework (issue: {issue}1263[#1263]) + +[float] +===== Bug fixes +- {pull}1252[#1252] Fix issue around setting `Recording` to `false` (issue: {issue}1250[#1250]) +- {pull}1259[#1259] ASP.NET: Move error capturing to Error event handler +- {pull}1305[#1305] Use Logger to log exception in AgentComponents initialization (issue: {issue}1254[#1254]) +- {pull}1311[#1311] Fix `NullReferenceException` in Elastic.Apm.Extensions.Logging(issue: {issue}1309[#1309]) + +[float] +===== Breaking changes +- {pull}1306[#1306] Do not capture HTTP child spans for Elasticsearch (issue: {issue}1276[#1276]) + [[release-notes-1.9.0]] ==== 1.9.0 diff --git a/src/Directory.Build.props b/src/Directory.Build.props index 996886293..cd8ca9697 100644 --- a/src/Directory.Build.props +++ b/src/Directory.Build.props @@ -2,12 +2,12 @@ - 1.9.0 - 1.9.0 - 1.9.0 - 1.9.0 + 1.10.0 + 1.10.0 + 1.10.0 + 1.10.0 Elastic and contributors - 2020 Elasticsearch BV + 2021 Elasticsearch BV https://github.com/elastic/apm-agent-dotnet true LICENSE From 468f44654c8ee88c5befeef1fb45a15bdfc03211 Mon Sep 17 00:00:00 2001 From: Gergely Kalapos Date: Fri, 28 May 2021 13:07:21 +0200 Subject: [PATCH 13/21] Update setup.asciidoc (#1318) fix typo --- docs/setup.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/setup.asciidoc b/docs/setup.asciidoc index 059e4fcb2..9afbb404b 100644 --- a/docs/setup.asciidoc +++ b/docs/setup.asciidoc @@ -473,7 +473,7 @@ A prerequisite for auto instrumentation with [`MongoDb.Driver`] is to configure ---- var settings = MongoClientSettings.FromConnectionString(mongoConnectionString); -settings.ClusterConfigurator = builder => builder.Subscribe(new MongoEventSubscriber()); +settings.ClusterConfigurator = builder => builder.Subscribe(new MongoDbEventSubscriber()); var mongoClient = new MongoClient(settings); ---- From deefeaf088fed50af6b7b2b0180983f443394515 Mon Sep 17 00:00:00 2001 From: Russ Cam Date: Tue, 1 Jun 2021 17:08:52 +1000 Subject: [PATCH 14/21] Don't package Elastic.Apm.Specification (#1316) This commit sets Elastic.Apm.Specification as a false project so that it is not packaged into a nuget package for release. Releasing from CI uses a constrained list in .ci/linux/deploy.sh to determine what can be released, but if there is a need to release from local, this constraint will not be adhered to. --- src/Elastic.Apm.Specification/Elastic.Apm.Specification.csproj | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Elastic.Apm.Specification/Elastic.Apm.Specification.csproj b/src/Elastic.Apm.Specification/Elastic.Apm.Specification.csproj index 78b5b6e69..654b2fca9 100644 --- a/src/Elastic.Apm.Specification/Elastic.Apm.Specification.csproj +++ b/src/Elastic.Apm.Specification/Elastic.Apm.Specification.csproj @@ -1,7 +1,8 @@ - netstandard2.0;net461 + netstandard2.0;net461 + false From 4dac7ac43f7f7b129e779847fa74653a09cf1d59 Mon Sep 17 00:00:00 2001 From: apmmachine <58790750+apmmachine@users.noreply.github.com> Date: Tue, 1 Jun 2021 04:39:44 -0400 Subject: [PATCH 15/21] synchronize json schema specs (#1320) Co-authored-by: apmmachine --- .../specs/metricset.json | 111 +++++++++++++++++- 1 file changed, 108 insertions(+), 3 deletions(-) diff --git a/src/Elastic.Apm.Specification/specs/metricset.json b/src/Elastic.Apm.Specification/specs/metricset.json index 13126665c..391ae3480 100644 --- a/src/Elastic.Apm.Specification/specs/metricset.json +++ b/src/Elastic.Apm.Specification/specs/metricset.json @@ -13,13 +13,118 @@ "object" ], "properties": { + "counts": { + "description": "Counts holds the bucket counts for histogram metrics. These numbers must be positive or zero. If Counts is specified, then Values is expected to be specified with the same number of elements, and with the same order.", + "type": [ + "null", + "array" + ], + "items": { + "type": "integer", + "minimum": 0 + }, + "minItems": 0 + }, + "type": { + "description": "Type holds an optional metric type: gauge, counter, or histogram. If Type is unknown, it will be ignored.", + "type": [ + "null", + "string" + ] + }, + "unit": { + "description": "Unit holds an optional unit for the metric. - \"percent\" (value is in the range [0,1]) - \"byte\" - a time unit: \"nanos\", \"micros\", \"ms\", \"s\", \"m\", \"h\", \"d\" If Unit is unknown, it will be ignored.", + "type": [ + "null", + "string" + ] + }, "value": { "description": "Value holds the value of a single metric sample.", - "type": "number" + "type": [ + "null", + "number" + ] + }, + "values": { + "description": "Values holds the bucket values for histogram metrics. Values must be provided in ascending order; failure to do so will result in the metric being discarded.", + "type": [ + "null", + "array" + ], + "items": { + "type": "number" + }, + "minItems": 0 } }, - "required": [ - "value" + "allOf": [ + { + "if": { + "properties": { + "counts": { + "type": "array" + } + }, + "required": [ + "counts" + ] + }, + "then": { + "properties": { + "values": { + "type": "array" + } + }, + "required": [ + "values" + ] + } + }, + { + "if": { + "properties": { + "values": { + "type": "array" + } + }, + "required": [ + "values" + ] + }, + "then": { + "properties": { + "counts": { + "type": "array" + } + }, + "required": [ + "counts" + ] + } + } + ], + "anyOf": [ + { + "properties": { + "value": { + "type": "number" + } + }, + "required": [ + "value" + ] + }, + { + "properties": { + "values": { + "type": "array" + } + }, + "required": [ + "values" + ] + } ] } } From 0d1ba83a137b12a192ce15fffd0eeac8a89a1f3c Mon Sep 17 00:00:00 2001 From: Russ Cam Date: Fri, 4 Jun 2021 11:13:57 +1000 Subject: [PATCH 16/21] Update context.destination.address (#1324) Relates: elastic/apm#426 --- src/Elastic.Apm.Azure.Storage/BlobUrl.cs | 2 +- .../AzureFileShareStorageDiagnosticListenerTests.cs | 2 +- .../AzureQueueStorageDiagnosticListenerTests.cs | 2 +- .../AzureStorageTestEnvironment.cs | 6 +++--- .../Elastic.Apm.Azure.Storage.Tests/BlobStorageTestsBase.cs | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/Elastic.Apm.Azure.Storage/BlobUrl.cs b/src/Elastic.Apm.Azure.Storage/BlobUrl.cs index cdd419baa..6eb87b19f 100644 --- a/src/Elastic.Apm.Azure.Storage/BlobUrl.cs +++ b/src/Elastic.Apm.Azure.Storage/BlobUrl.cs @@ -25,7 +25,7 @@ internal abstract class StorageUrl protected StorageUrl(Uri url) { StorageAccountName = url.Host.Split(SplitDomain, 2)[0]; - FullyQualifiedNamespace = url.GetLeftPart(UriPartial.Authority) + "/"; + FullyQualifiedNamespace = url.Host; } public string StorageAccountName { get; } diff --git a/test/Elastic.Apm.Azure.Storage.Tests/AzureFileShareStorageDiagnosticListenerTests.cs b/test/Elastic.Apm.Azure.Storage.Tests/AzureFileShareStorageDiagnosticListenerTests.cs index 9b201fb67..461a67067 100644 --- a/test/Elastic.Apm.Azure.Storage.Tests/AzureFileShareStorageDiagnosticListenerTests.cs +++ b/test/Elastic.Apm.Azure.Storage.Tests/AzureFileShareStorageDiagnosticListenerTests.cs @@ -176,7 +176,7 @@ private void AssertSpan(string action, string resource) span.Context.Destination.Should().NotBeNull(); var destination = span.Context.Destination; - destination.Address.Should().Be(_environment.StorageAccountConnectionStringProperties.FileUrl); + destination.Address.Should().Be(_environment.StorageAccountConnectionStringProperties.FileFullyQualifiedNamespace); destination.Service.Name.Should().Be(AzureFileStorage.SubType); destination.Service.Resource.Should().Be($"{AzureFileStorage.SubType}/{_environment.StorageAccountConnectionStringProperties.AccountName}"); destination.Service.Type.Should().Be(ApiConstants.TypeStorage); diff --git a/test/Elastic.Apm.Azure.Storage.Tests/AzureQueueStorageDiagnosticListenerTests.cs b/test/Elastic.Apm.Azure.Storage.Tests/AzureQueueStorageDiagnosticListenerTests.cs index 9f6c2beb2..0d279fc51 100644 --- a/test/Elastic.Apm.Azure.Storage.Tests/AzureQueueStorageDiagnosticListenerTests.cs +++ b/test/Elastic.Apm.Azure.Storage.Tests/AzureQueueStorageDiagnosticListenerTests.cs @@ -98,7 +98,7 @@ private void AssertSpan(string action, string queueName) span.Context.Destination.Should().NotBeNull(); var destination = span.Context.Destination; - destination.Address.Should().Be(_environment.StorageAccountConnectionStringProperties.QueueUrl); + destination.Address.Should().Be(_environment.StorageAccountConnectionStringProperties.QueueFullyQualifiedNamespace); destination.Service.Name.Should().Be(AzureQueueStorage.SubType); destination.Service.Resource.Should().Be($"{AzureQueueStorage.SubType}/{queueName}"); destination.Service.Type.Should().Be(ApiConstants.TypeMessaging); diff --git a/test/Elastic.Apm.Azure.Storage.Tests/AzureStorageTestEnvironment.cs b/test/Elastic.Apm.Azure.Storage.Tests/AzureStorageTestEnvironment.cs index dfcd945c6..9fc7faa1d 100644 --- a/test/Elastic.Apm.Azure.Storage.Tests/AzureStorageTestEnvironment.cs +++ b/test/Elastic.Apm.Azure.Storage.Tests/AzureStorageTestEnvironment.cs @@ -112,10 +112,10 @@ public StorageAccountProperties(string defaultEndpointsProtocol, string accountN public string DefaultEndpointsProtocol { get; } - public string QueueUrl => $"{DefaultEndpointsProtocol}://{AccountName}.queue.{EndpointSuffix}/"; + public string QueueFullyQualifiedNamespace => $"{AccountName}.queue.{EndpointSuffix}"; - public string BlobUrl => $"{DefaultEndpointsProtocol}://{AccountName}.blob.{EndpointSuffix}/"; + public string BlobFullyQualifiedNamespace => $"{AccountName}.blob.{EndpointSuffix}"; - public string FileUrl => $"{DefaultEndpointsProtocol}://{AccountName}.file.{EndpointSuffix}/"; + public string FileFullyQualifiedNamespace => $"{AccountName}.file.{EndpointSuffix}"; } } diff --git a/test/Elastic.Apm.Azure.Storage.Tests/BlobStorageTestsBase.cs b/test/Elastic.Apm.Azure.Storage.Tests/BlobStorageTestsBase.cs index 1fa8d0cd3..a415b6d3d 100644 --- a/test/Elastic.Apm.Azure.Storage.Tests/BlobStorageTestsBase.cs +++ b/test/Elastic.Apm.Azure.Storage.Tests/BlobStorageTestsBase.cs @@ -43,7 +43,7 @@ protected void AssertSpan(string action, string resource, int count = 1) span.Context.Destination.Should().NotBeNull(); var destination = span.Context.Destination; - destination.Address.Should().Be(Environment.StorageAccountConnectionStringProperties.BlobUrl); + destination.Address.Should().Be(Environment.StorageAccountConnectionStringProperties.BlobFullyQualifiedNamespace); destination.Service.Name.Should().Be(AzureBlobStorage.SubType); destination.Service.Resource.Should().Be($"{AzureBlobStorage.SubType}/{Environment.StorageAccountConnectionStringProperties.AccountName}"); destination.Service.Type.Should().Be(ApiConstants.TypeStorage); From 242ee4b9c86155dae35667eea7d29c9ef8f2dc13 Mon Sep 17 00:00:00 2001 From: Russ Cam Date: Fri, 4 Jun 2021 11:20:38 +1000 Subject: [PATCH 17/21] Update docs (#1327) --- docs/configuration.asciidoc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/configuration.asciidoc b/docs/configuration.asciidoc index 71d626c82..c98337614 100644 --- a/docs/configuration.asciidoc +++ b/docs/configuration.asciidoc @@ -33,7 +33,6 @@ using Elastic.Apm.AspNetCore; public class Startup { - private readonly IConfiguration _configuration; public Startup(IConfiguration configuration) @@ -44,7 +43,7 @@ public class Startup public void Configure(IApplicationBuilder app, IHostingEnvironment env) { //Registers the agent with an IConfiguration instance: - app.UseElasticApm(Configuration); + app.UseElasticApm(_configuration); //Rest of the Configure() method... } From 740403f0adab4d8a150d7b4b58faa2f5f248d27f Mon Sep 17 00:00:00 2001 From: Russ Cam Date: Fri, 4 Jun 2021 13:39:32 +1000 Subject: [PATCH 18/21] Mark MicrosoftAzureBlobStorageTracer internal (#1326) This commit marks MicrosoftAzureBlobStorageTracer as an internal type; it should not be public. --- .../MicrosoftAzureBlobStorageTracer.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Elastic.Apm.Azure.Storage/MicrosoftAzureBlobStorageTracer.cs b/src/Elastic.Apm.Azure.Storage/MicrosoftAzureBlobStorageTracer.cs index a6ba95202..2064323a6 100644 --- a/src/Elastic.Apm.Azure.Storage/MicrosoftAzureBlobStorageTracer.cs +++ b/src/Elastic.Apm.Azure.Storage/MicrosoftAzureBlobStorageTracer.cs @@ -14,7 +14,7 @@ namespace Elastic.Apm.Azure.Storage /// /// Creates HTTP spans wth Azure Blob storage details from Microsoft.Azure.Storage.Blob /// - public class MicrosoftAzureBlobStorageTracer : IHttpSpanTracer + internal class MicrosoftAzureBlobStorageTracer : IHttpSpanTracer { public bool IsMatch(string method, Uri requestUrl, Func headerGetter) => requestUrl.Host.EndsWith(".blob.core.windows.net", StringComparison.Ordinal) || From 80b08ece54712022826b56c60d5dfd71f354f4c4 Mon Sep 17 00:00:00 2001 From: Russ Cam Date: Mon, 7 Jun 2021 17:19:41 +1000 Subject: [PATCH 19/21] Update README (#1325) This commit updates the README to add details of new packages in 1.10 --- README.md | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index a891cbdcb..7d76a576d 100644 --- a/README.md +++ b/README.md @@ -15,12 +15,15 @@ Official NuGet packages can be referenced from [NuGet.org](https://www.nuget.org | `Elastic.Apm` | The core of the Agent, Public Agent API, Auto instrumentation for libraries that are part of .NET Standard 2.0. | [![NuGet Release][ElasticApm-image]][ElasticApm-nuget-url] | | `Elastic.Apm.AspNetCore` | ASP.NET Core auto instrumentation. | [![NuGet Release][ElasticApmAspNetCore-image]][ElasticApmAspNetCore-nuget-url] | | `Elastic.Apm.EntityFrameworkCore` | Entity Framework Core auto instrumentation. | [![NuGet Release][Elastic.Apm.EntityFrameworkCore-image]][Elastic.Apm.EntityFrameworkCore-nuget-url] | -| `Elastic.Apm.NetCoreAll` | References every .NET Core related elastic APM package. It can be used to simply turn on the agent with a single line and activate all auto instrumentation. | [![NuGet Release][Elastic.Apm.NetCoreAll-image]][Elastic.Apm.NetCoreAll-nuget-url] | +| `Elastic.Apm.NetCoreAll` | References every .NET Core related Elastic APM package. It can be used to simply turn on the agent and activate all auto instrumentation. | [![NuGet Release][Elastic.Apm.NetCoreAll-image]][Elastic.Apm.NetCoreAll-nuget-url] | | `Elastic.Apm.AspNetFullFramework` | ASP.NET (classic) auto instrumentation with an IIS Module. | [![NuGet Release][Elastic.Apm.AspNetFullFramework-image]][Elastic.Apm.AspNetFullFramework-nuget-url] | | `Elastic.Apm.EntityFramework6` | Entity Framework 6 auto instrumentation. | [![NuGet Release][Elastic.Apm.EntityFramework6-image]][Elastic.Apm.EntityFramework6-nuget-url] | | `Elastic.Apm.SqlClient` | `System.Data.SqlClient` and `Microsoft.Data.SqlClient` auto instrumentation. [More details](/src/Elastic.Apm.SqlClient/README.md) | [![NuGet Release][Elastic.Apm.SqlClient-image]][Elastic.Apm.SqlClient-nuget-url] | | `Elastic.Apm.Elasticsearch` | Integration with the .NET clients for Elasticsearch. | [![NuGet Release][Elastic.Apm.Elasticsearch-image]][Elastic.Apm.Elasticsearch-nuget-url] | | `Elastic.Apm.StackExchange.Redis` | Integration with the StackExchange.Redis client for Redis. | [![NuGet Release][Elastic.Apm.StackExchange.Redis-image]][Elastic.Apm.StackExchange.Redis-nuget-url] | +| `Elastic.Apm.MongoDb` | Integration with the MongoDb.Driver driver for MongoDb. | [![NuGet Release][Elastic.Apm.MongoDb-image]][Elastic.Apm.MongoDb-nuget-url] | +| `Elastic.Apm.Azure.ServiceBus` | Integration with Azure ServiceBus | [![NuGet Release][Elastic.Apm.Azure.ServiceBus-image]][Elastic.Apm.Azure.ServiceBus-nuget-url] | +| `Elastic.Apm.Azure.Storage` | Integration with Azure Storage | [![NuGet Release][Elastic.Apm.Azure.Storage-image]][Elastic.Apm.Azure.Storage-nuget-url] | ## Documentation @@ -56,6 +59,9 @@ These are the main folders within the repository: * `Elastic.Apm.SqlClient`: Auto-instrumentation for `System.Data.SqlClient` and `Microsoft.Data.SqlClient`. * `Elastic.Apm.Elasticsearch`: Auto-instrumentation for the official .NET clients for Elasticsearch. * `Elastic.Apm.StackExchange.Redis`: Auto-instrumentation for the StackExchange.Redis client for Redis. + * `Elastic.Apm.MongoDb`: Instrumentation for the MongoDb.Driver driver for MongoDb. + * `Elastic.Apm.Azure.ServiceBus`: Instrumentation for Azure ServiceBus. + * `Elastic.Apm.Azure.Storage`: Instrumentation for Azure Storage. * `test`: This folder contains test projects. Typically each project from the `src` folder has a corresponding test project. * `Elastic.Apm.Tests`: Tests the `Elastic.Apm` project. * `Elastic.Apm.AspNetCore.Tests`: Tests the `Elastic.Apm.AspNetCore` project. @@ -106,3 +112,15 @@ https://img.shields.io/nuget/v/Elastic.Apm.Elasticsearch.svg [Elastic.Apm.StackExchange.Redis-nuget-url]:https://www.nuget.org/packages/Elastic.Apm.StackExchange.Redis/ [Elastic.Apm.StackExchange.Redis-image]: https://img.shields.io/nuget/v/Elastic.Apm.StackExchange.Redis.svg + +[Elastic.Apm.MongoDb-nuget-url]:https://www.nuget.org/packages/Elastic.Apm.MongoDb/ +[Elastic.Apm.MongoDb-image]: +https://img.shields.io/nuget/v/Elastic.Apm.MongoDb.svg + +[Elastic.Apm.Azure.ServiceBus-nuget-url]:https://www.nuget.org/packages/Elastic.Apm.Azure.ServiceBus/ +[Elastic.Apm.Azure.ServiceBus-image]: +https://img.shields.io/nuget/v/Elastic.Apm.Azure.ServiceBus.svg + +[Elastic.Apm.Azure.Storage-nuget-url]:https://www.nuget.org/packages/Elastic.Apm.Azure.Storage/ +[Elastic.Apm.Azure.Storage-image]: +https://img.shields.io/nuget/v/Elastic.Apm.Azure.Storage.svg From 04c147daa8bdadcc098a311c4916be387e60990a Mon Sep 17 00:00:00 2001 From: Russ Cam Date: Tue, 8 Jun 2021 11:42:02 +1000 Subject: [PATCH 20/21] fix spacing and cross references in docs (#1328) --- docs/troubleshooting.asciidoc | 26 +++++++++++++------------- docs/upgrading.asciidoc | 20 +++++++++++--------- 2 files changed, 24 insertions(+), 22 deletions(-) diff --git a/docs/troubleshooting.asciidoc b/docs/troubleshooting.asciidoc index d55cef742..73878e658 100644 --- a/docs/troubleshooting.asciidoc +++ b/docs/troubleshooting.asciidoc @@ -68,15 +68,15 @@ named my_log_file.log: - - <1> - - - - - + + <1> + + + + + ---- @@ -186,17 +186,17 @@ set ELASTIC_APM_STARTUP_HOOKS_LOGGING=1 ---- and then running the application in a context where the environment variable will be visible. In setting this value, -an `ElasticApmAgentStartupHook.log` file is written to in the directory containing the startup hook assembly, in addition to +an `ElasticApmAgentStartupHook.log` file is written to the directory containing the startup hook assembly, in addition to writing to standard output. [float] [[agent-overhead]] === The agent causes too much overhead -A good place to start is [config-all-options-summary]. There are multiple settings with the `Performance` keyword which can help you tweak the agent for your needs. +A good place to start is <>. There are multiple settings with the `Performance` keyword which can help you tweak the agent for your needs. The most expensive operation in the agent is typically stack trace capturing. The agent, by default, only captures stack traces for spans with a duration of 5ms or more, and with a limit of 50 stack frames. If this is too much in your environment, consider disabling stack trace capturing either partially or entirely: -- To disable stack trace capturing for spans, but continue to capture stack traces for errors, set the [config-span-frames-min-duration] to `0` and leave the [config-stack-trace-limit] on its default. -- To disable stack trace capturing entirely –which in most applications reduces the agent overhead dramatically– set [config-stack-trace-limit] to `0`. \ No newline at end of file +- To disable stack trace capturing for spans, but continue to capture stack traces for errors, set the <> to `0` and leave the <> on its default. +- To disable stack trace capturing entirely –which in most applications reduces the agent overhead dramatically– set <> to `0`. \ No newline at end of file diff --git a/docs/upgrading.asciidoc b/docs/upgrading.asciidoc index 219c6abcf..da51fa1cd 100644 --- a/docs/upgrading.asciidoc +++ b/docs/upgrading.asciidoc @@ -21,13 +21,15 @@ The table below is a simplified description of this policy. [options="header"] |==== |Agent version |EOL Date |Maintained until -|1.8.x |2022-08-17 |1.9.0 -|1.7.x |2022-05-12 |1.8.0 -|1.6.x |2022-01-10 |1.7.0 -|1.5.x |2021-11-09 |1.6.0 -|1.4.x |2021-09-20 |1.5.0 -|1.3.x |2021-08-12 |1.4.0 -|1.2.x |2021-05-22 |1.3.0 -|1.1.x |2021-04-01 |1.2.0 -|1.0.x |2021-01-31 |1.1.0 +|1.10.x |2022-11-28 |1.11.0 +|1.9.x |2022-10-07 |1.10.0 +|1.8.x |2022-08-17 |1.9.0 +|1.7.x |2022-05-12 |1.8.0 +|1.6.x |2022-01-10 |1.7.0 +|1.5.x |2021-11-09 |1.6.0 +|1.4.x |2021-09-20 |1.5.0 +|1.3.x |2021-08-12 |1.4.0 +|1.2.x |2021-05-22 |1.3.0 +|1.1.x |2021-04-01 |1.2.0 +|1.0.x |2021-01-31 |1.1.0 |==== From d924d8dee4e203055cc27447d249ae96f7d03b5d Mon Sep 17 00:00:00 2001 From: Gergely Kalapos Date: Tue, 8 Jun 2021 07:57:56 +0200 Subject: [PATCH 21/21] Prefer W3C traceparent over elastic-apm-traceparent (#1302) * Update WebRequestTransactionCreator.cs Prefer W3C traceparent over elastic-apm-traceparent * Update DistributedTracingAspNetCoreTests.cs Add test to assert that the traceparent header is used over elastic-apm-traceparent --- .../WebRequestTransactionCreator.cs | 10 +++++----- .../DistributedTracingAspNetCoreTests.cs | 18 ++++++++++++++++++ 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/src/Elastic.Apm.AspNetCore/WebRequestTransactionCreator.cs b/src/Elastic.Apm.AspNetCore/WebRequestTransactionCreator.cs index 67e1f9814..1025a84d5 100644 --- a/src/Elastic.Apm.AspNetCore/WebRequestTransactionCreator.cs +++ b/src/Elastic.Apm.AspNetCore/WebRequestTransactionCreator.cs @@ -36,12 +36,12 @@ internal static ITransaction StartTransactionAsync(HttpContext context, IApmLogg ITransaction transaction; var transactionName = $"{context.Request.Method} {context.Request.Path}"; - var containsPrefixedTraceParentHeader = - context.Request.Headers.TryGetValue(TraceContext.TraceParentHeaderNamePrefixed, out var traceParentHeader); + var containsTraceParentHeader = + context.Request.Headers.TryGetValue(TraceContext.TraceParentHeaderName, out var traceParentHeader); - var containsTraceParentHeader = false; - if (!containsPrefixedTraceParentHeader) - containsTraceParentHeader = context.Request.Headers.TryGetValue(TraceContext.TraceParentHeaderName, out traceParentHeader); + var containsPrefixedTraceParentHeader = false; + if (!containsTraceParentHeader) + containsPrefixedTraceParentHeader = context.Request.Headers.TryGetValue(TraceContext.TraceParentHeaderNamePrefixed, out traceParentHeader); if (containsPrefixedTraceParentHeader || containsTraceParentHeader) { diff --git a/test/Elastic.Apm.AspNetCore.Tests/DistributedTracingAspNetCoreTests.cs b/test/Elastic.Apm.AspNetCore.Tests/DistributedTracingAspNetCoreTests.cs index ae16b40d5..d95bbe888 100644 --- a/test/Elastic.Apm.AspNetCore.Tests/DistributedTracingAspNetCoreTests.cs +++ b/test/Elastic.Apm.AspNetCore.Tests/DistributedTracingAspNetCoreTests.cs @@ -183,6 +183,24 @@ public async Task DistributedTraceAcross2ServicesWithTraceState() _payloadSender2.FirstTransaction.Context.Request.Headers["tracestate"].Should().Be("rojo=00f067aa0ba902b7,congo=t61rcWkgMzE"); } + /// + /// Makes sure that the header `traceparent` is used when both `traceparent` and `elastic-apm-traceparent` are present. + /// + [Fact] + public async Task PreferW3CTraceHeaderOverElasticTraceHeader() + { + var client = new HttpClient(); + var expectedTraceId = "0af7651916cd43dd8448eb211c80319c"; + var expectedParentId = "b7ad6b7169203331"; + client.DefaultRequestHeaders.Add("traceparent", $"00-{expectedTraceId}-{expectedParentId}-01"); + client.DefaultRequestHeaders.Add("elastic-apm-traceparent", "00-000000000000000000000000000019c-0000000000000001-01"); + var res = await client.GetAsync("http://localhost:5901/Home/Index"); + res.IsSuccessStatusCode.Should().BeTrue(); + + _payloadSender1.FirstTransaction.TraceId.Should().Be(expectedTraceId); + _payloadSender1.FirstTransaction.ParentId.Should().Be(expectedParentId); + } + public async Task DisposeAsync() { _cancellationTokenSource.Cancel();