Breakdown metrics (#1271)

* Add breakdown metrics - heavily WIP

* Breakdown: cleanup, send transaction.breakdown.count

* Update Span.cs

* Add more breakdown tests

* Add span.self_time to DisableMetrics

* Update BreakdownTests.cs

* Update metrics.asciidoc

Add breakdown docs

* Update metrics.asciidoc

* Add handling for spans with type `app`

* Add more breakdown tests

* Make breakdown self-time microseconds

* Do not pass BreakdownMetricsProvider to Transactions and Spans

* Add SpanTypeAndSubtype and handling disable breakdown in AgentComponents

* Add IMetricsProvider.IsEnabled

* Move BreakdownMetricsProvider.CaptureTransaction to Transaction.cs

Also review feedback:
- Some naming changes
- Handling 1K breakdown metricset limit (including test)

* Update BreakdownMetricsProvider.cs

Only log once

* Update CgroupMetricsProviderTests.cs

Adapting assets to the changed metrics API

* Update docs/metrics.asciidoc

Co-authored-by: Brandon Morelli <bmorelli25@gmail.com>

* Address failing tests

* Address failing test

* Update MockPayloadSender.cs

Remove lock around `WaitHandle.WaitOne()`

* Update Span.cs

Move SpanTiming calculation before `_payloadSender.QueueSpan`.

* Update BreakdownTests.cs

* Update BreakdownTests.cs

* Update BreakdownTests.cs

* Update BreakdownTests.cs

* Update BreakdownTests.cs

* Update BreakdownTests.cs

* Update BreakdownTests.cs

* Update BreakdownTests.cs

Co-authored-by: Brandon Morelli <bmorelli25@gmail.com>

docs/metrics.asciidoc

@@ -174,4 +174,71 @@ format: ms
 Platform: all.
 
 The approximate accumulated collection elapsed time in milliseconds.
---
+--
+
+[float]
+[[metrics-application]]
+=== Built-in application metrics
+
+To power the {apm-app-ref}/transactions.html[Time spent by span type] graph,
+the agent collects summarized metrics about the timings of spans and transactions,
+broken down by span type.
+
+*`transaction.duration`*::
++
+--
+type: simple timer
+
+This timer tracks the duration of transactions and allows for the creation of graphs displaying a weighted average.
+
+Fields:
+
+* `sum.us`: The sum of all transaction durations in microseconds since the last report (the delta)
+* `count`: The count of all transactions since the last report (the delta)
+
+You can filter and group by these dimensions:
+
+* `transaction.name`: The name of the transaction
+* `transaction.type`: The type of the transaction, for example `request`
+
+--
+
+
+*`transaction.breakdown.count`*::
++
+--
+type: long
+
+format: count (delta)
+
+The number of transactions for which breakdown metrics (`span.self_time`) have been created.
+As the Java agent tracks the breakdown for both sampled and non-sampled transactions,
+this metric is equivalent to `transaction.duration.count`
+
+You can filter and group by these dimensions:
+
+* `transaction.name`: The name of the transaction
+* `transaction.type`: The type of the transaction, for example `request`
+
+--
+
+*`span.self_time`*::
++
+--
+type: simple timer
+
+This timer tracks the span self-times and is the basis of the transaction breakdown visualization.
+
+Fields:
+
+* `sum.us`: The sum of all span self-times in microseconds since the last report (the delta)
+* `count`: The count of all span self-times since the last report (the delta)
+
+You can filter and group by these dimensions:
+
+* `transaction.name`: The name of the transaction
+* `transaction.type`: The type of the transaction, for example `request`
+* `span.type`: The type of the span, for example `app`, `template` or `db`
+* `span.subtype`: The sub-type of the span, for example `mysql` (optional)
+
+--

src/Elastic.Apm/AgentComponents.cs

@@ -10,6 +10,7 @@
 using Elastic.Apm.Helpers;
 using Elastic.Apm.Logging;
 using Elastic.Apm.Metrics;
+using Elastic.Apm.Metrics.MetricsProvider;
 using Elastic.Apm.Report;
 using Elastic.Apm.ServerInfo;
 
@@ -30,7 +31,8 @@ internal AgentComponents(
 			IMetricsCollector metricsCollector,
 			ICurrentExecutionSegmentsContainer currentExecutionSegmentsContainer,
 			ICentralConfigFetcher centralConfigFetcher,
-			IApmServerInfo apmServerInfo
+			IApmServerInfo apmServerInfo,
+			BreakdownMetricsProvider breakdownMetricsProvider = null
 		)
 		{
 			try
@@ -53,17 +55,19 @@ IApmServerInfo apmServerInfo
 
 				HttpTraceConfiguration = new HttpTraceConfiguration();
 
-				TracerInternal = new Tracer(Logger, Service, PayloadSender, ConfigStore,
-					currentExecutionSegmentsContainer ?? new CurrentExecutionSegmentsContainer(), ApmServerInfo);
-
 				if (ConfigurationReader.Enabled)
 				{
+					breakdownMetricsProvider ??= new BreakdownMetricsProvider(Logger);
+
 					CentralConfigFetcher = centralConfigFetcher ?? new CentralConfigFetcher(Logger, ConfigStore, Service);
-					MetricsCollector = metricsCollector ?? new MetricsCollector(Logger, PayloadSender, ConfigStore);
+					MetricsCollector = metricsCollector ?? new MetricsCollector(Logger, PayloadSender, ConfigStore, breakdownMetricsProvider);
 					MetricsCollector.StartCollecting();
 				}
 				else
 					Logger.Info()?.Log("The Elastic APM .NET Agent is disabled - the agent won't capture traces and metrics.");
+
+				TracerInternal = new Tracer(Logger, Service, PayloadSender, ConfigStore,
+					currentExecutionSegmentsContainer ?? new CurrentExecutionSegmentsContainer(), ApmServerInfo, breakdownMetricsProvider);
 			}
 			catch (Exception e)
 			{

src/Elastic.Apm/Api/Tracer.cs

@@ -11,6 +11,7 @@
 using Elastic.Apm.Config;
 using Elastic.Apm.Helpers;
 using Elastic.Apm.Logging;
+using Elastic.Apm.Metrics.MetricsProvider;
 using Elastic.Apm.Model;
 using Elastic.Apm.Report;
 using Elastic.Apm.ServerInfo;
@@ -24,14 +25,16 @@ internal class Tracer : ITracer
 		private readonly ScopedLogger _logger;
 		private readonly IPayloadSender _sender;
 		private readonly Service _service;
+		private readonly BreakdownMetricsProvider _breakdownMetricsProvider;
 
 		public Tracer(
 			IApmLogger logger,
 			Service service,
 			IPayloadSender payloadSender,
 			IConfigSnapshotProvider configProvider,
 			ICurrentExecutionSegmentsContainer currentExecutionSegmentsContainer,
-			IApmServerInfo apmServerInfo
+			IApmServerInfo apmServerInfo,
+			BreakdownMetricsProvider breakdownMetricsProvider
 		)
 		{
 			_logger = logger?.Scoped(nameof(Tracer));
@@ -41,6 +44,7 @@ IApmServerInfo apmServerInfo
 			CurrentExecutionSegmentsContainer = currentExecutionSegmentsContainer.ThrowIfArgumentNull(nameof(currentExecutionSegmentsContainer));
 			DbSpanCommon = new DbSpanCommon(logger);
 			_apmServerInfo = apmServerInfo;
+			_breakdownMetricsProvider = breakdownMetricsProvider;
 		}
 
 		internal ICurrentExecutionSegmentsContainer CurrentExecutionSegmentsContainer { get; }
@@ -61,16 +65,27 @@ public ITransaction StartTransaction(string name, string type, DistributedTracin
 			return new NoopTransaction(name, type, CurrentExecutionSegmentsContainer);
 		}
 
+		internal Transaction StartTransactionInternal(string name, string type,
+			long? timestamp = null
+		)
+			=> StartTransactionInternal(name, type, null, false, timestamp);
+
 		private Transaction StartTransactionInternal(string name, string type, DistributedTracingData distributedTracingData = null,
-			bool ignoreActivity = false
+			bool ignoreActivity = false, long? timestamp = null
 		)
 		{
 			var currentConfig = _configProvider.CurrentSnapshot;
 			var retVal = new Transaction(_logger, name, type, new Sampler(currentConfig.TransactionSampleRate), distributedTracingData
-				, _sender, currentConfig, CurrentExecutionSegmentsContainer, _apmServerInfo, ignoreActivity) { Service = _service };
+				, _sender, currentConfig, CurrentExecutionSegmentsContainer, _apmServerInfo, _breakdownMetricsProvider, ignoreActivity, timestamp)
+			{
+				Service = _service
+			};
+
+
 
 			_logger.Debug()?.Log("Starting {TransactionValue}", retVal);
 			return retVal;
+
 		}
 
 		public void CaptureTransaction(string name, string type, Action<ITransaction> action, DistributedTracingData distributedTracingData = null)

src/Elastic.Apm/Metrics/IMetricsProvider.cs

@@ -1,9 +1,10 @@
-// Licensed to Elasticsearch B.V under one or more agreements.
+// Licensed to Elasticsearch B.V under
+// one or more agreements.
 // Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
 // See the LICENSE file in the project root for more information
 
 using System.Collections.Generic;
-using Elastic.Apm.Api;
+using Elastic.Apm.Helpers;
 
 namespace Elastic.Apm.Metrics
 {
@@ -27,22 +28,32 @@ internal interface IMetricsProvider
 		/// </summary>
 		string DbgName { get; }
 
-		/// <summary>
-		/// The main part of the provider, the implementor should do the work to read the value(s) of the given metric(s) in this
-		/// method.
-		/// </summary>
-		/// <returns>The key and the value of the metric(s)</returns>
-		IEnumerable<MetricSample> GetSamples();
-
 		/// <summary>
 		/// Indicates if metrics were already collected - or there was an attempt to collect them.
 		/// Until this property is false, metrics from the implementor won't be collected.
 		/// This property exists to cover cases when the metric collection happens in the background
 		/// (e.g. collecting GC metrics through EventListener) and values are not captured directly in
-		/// the <see cref="GetSamples"/> method.
-		/// If metrics are captured on the fly in <see cref="GetSamples"/> just set this to <code>true</code>
+		/// the <see cref="GetSamples" /> method.
+		/// If metrics are captured on the fly in <see cref="GetSamples" /> just set this to <code>true</code>
 		/// during initialization.
 		/// </summary>
 		bool IsMetricAlreadyCaptured { get; }
+
+		/// <summary>
+		/// The main part of the provider, the implementor should do the work to read the value(s) of the given metric(s) in this
+		/// method.
+		/// </summary>
+		/// <returns>The key and the value of the metric(s)</returns>
+		IEnumerable<MetricSet> GetSamples();
+
+		/// <summary>
+		/// Indicates whether this instance is enabled to collect metrics based on the DisableMetrics config.
+		/// The implementor must match the <paramref name="disabledMetrics"></paramref> against metrics which it collects.
+		/// If the implementor collects multiple metrics, this method returns <code>true</code> if any of the metrics it
+		/// collects is enabled, <code>false</code> otherwise.
+		/// </summary>
+		/// <param name="disabledMetrics">The list of disabled metrics</param>
+		/// <returns><code>true</code> if any of the metrics collected is enabled, <code>false</code> otherwise.</returns>
+		bool IsEnabled(IReadOnlyList<WildcardMatcher> disabledMetrics);
 	}
 }

src/Elastic.Apm/Metrics/MetricSet.cs

@@ -4,6 +4,7 @@
 
 using System.Collections.Generic;
 using Elastic.Apm.Api;
+using Elastic.Apm.Api.Constraints;
 using Elastic.Apm.Report.Serialization;
 using Elastic.Apm.Libraries.Newtonsoft.Json;
 
@@ -20,5 +21,25 @@ public MetricSet(long timestamp, IEnumerable<MetricSample> samples)
 
 		/// <inheritdoc />
 		public long Timestamp { get; set; }
+
+		public TransactionInfo Transaction { get; set; }
+
+		public SpanInfo Span { get; set; }
+	}
+
+	internal class TransactionInfo
+	{
+		[MaxLength]
+		public string Name { get; set; }
+		[MaxLength]
+		public string Type { get; set; }
+	}
+
+	internal class SpanInfo
+	{
+		[MaxLength]
+		public string Type { get; set; }
+		[MaxLength]
+		public string SubType { get; set; }
 	}
 }