Propagate sample rate through tracestate

This commit propagates the sample rate of the
APM agent through the tracestate header, and
uses the sample rate received in incoming tracestate
as the sample rate for non-root transactions and spans.

Introduce TraceState type to model tracestate,
allowing incoming headers to be added and sample
rate to be set. Move TraceState validation from
TraceContext onto TraceState.

Obsolete TryDeserializeFromString and SerializeToString
on DistributedTracingData in favour of a method that accepts
both traceparent and tracestate, and properties that get
the traceparent and tracestate text headers, respectively.

Change Transaction and Span SampleRate to double?
to allow a value to be omitted when incoming
Trace context does not contain a tracestate header
or the header does not contain the es vendor s attribute.

Closes #1021

sample/ApiSamples/ApiSamples.csproj

@@ -2,7 +2,7 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFrameworks>netcoreapp2.1;netcoreapp3.0</TargetFrameworks>
+    <TargetFrameworks>netcoreapp2.1;netcoreapp3.0;net5.0</TargetFrameworks>
     <IsPackable>false</IsPackable>
   </PropertyGroup>
   <ItemGroup>

sample/ApiSamples/Program.cs

@@ -3,28 +3,29 @@
 // See the LICENSE file in the project root for more information
 
 using System;
-#if NETCOREAPP3_0
 using System.Collections.Generic;
-#endif
 using System.Diagnostics;
+using System.Reflection;
 using System.Threading;
 using Elastic.Apm;
 using Elastic.Apm.Api;
 
 namespace ApiSamples
 {
 	/// <summary>
-	/// This class exercices the Public Agent API.
+	/// This class exercises the Public Agent API.
 	/// </summary>
 	internal class Program
 	{
 		private static void Main(string[] args)
 		{
-			if (args.Length == 1) //in case it's started with an argument we try to parse the argument as a DistributedTracingData
+			if (args.Length == 2) //in case it's started with arguments, parse DistributedTracingData from them
 			{
-				WriteLineToConsole($"Callee process started - continuing trace with distributed tracing data: {args[0]}");
-				var transaction2 = Agent.Tracer.StartTransaction("Transaction2", "TestTransaction",
-					DistributedTracingData.TryDeserializeFromString(args[0]));
+				var distributedTracingData =
+					DistributedTracingData.FromTraceContext(args[0], args[1]);
+
+				WriteLineToConsole($"Callee process started - continuing trace with distributed tracing data: {distributedTracingData}");
+				var transaction2 = Agent.Tracer.StartTransaction("Transaction2", "TestTransaction", distributedTracingData);
 
 				try
 				{
@@ -35,7 +36,7 @@ private static void Main(string[] args)
 					transaction2.End();
 				}
 
-				Thread.Sleep(1000);
+				Thread.Sleep(TimeSpan.FromSeconds(11));
 				WriteLineToConsole("About to exit");
 			}
 			else
@@ -45,7 +46,7 @@ private static void Main(string[] args)
 
 				//WIP: if the process terminates the agent
 				//potentially does not have time to send the transaction to the server.
-				Thread.Sleep(1000);
+				Thread.Sleep(TimeSpan.FromSeconds(11));
 
 				WriteLineToConsole("About to exit - press any key...");
 				Console.ReadKey();
@@ -160,14 +161,23 @@ public static void PassDistributedTracingData()
 
 				//We start the sample app again with a new service name and we pass DistributedTracingData to it
 				//In the main method we check for this and continue the trace.
-				var startInfo = new ProcessStartInfo();
-				startInfo.Environment["ELASTIC_APM_SERVICE_NAME"] = "Service2";
-				var outgoingDistributedTracingData = (Agent.Tracer.CurrentSpan?.OutgoingDistributedTracingData
-					?? Agent.Tracer.CurrentTransaction?.OutgoingDistributedTracingData)?.SerializeToString();
-				startInfo.FileName = "dotnet";
-				startInfo.Arguments = $"run {outgoingDistributedTracingData}";
+
+				var distributedTracingData = Agent.Tracer.CurrentSpan?.OutgoingDistributedTracingData
+					?? Agent.Tracer.CurrentTransaction?.OutgoingDistributedTracingData;
+
+				var traceParent = distributedTracingData?.TraceparentTextHeader;
+				var traceState = distributedTracingData?.TracestateTextHeader;
+				var assembly = Assembly.GetExecutingAssembly().Location;
+
 				WriteLineToConsole(
-					$"Spawning callee process and passing outgoing distributed tracing data: {outgoingDistributedTracingData} to it...");
+					$"Spawning callee process and passing outgoing distributed tracing data: {traceParent} {traceState} to it...");
+				var startInfo = new ProcessStartInfo
+				{
+					FileName = "dotnet",
+					Arguments = $"{assembly} {traceParent} {traceState}"
+				};
+
+				startInfo.Environment["ELASTIC_APM_SERVICE_NAME"] = "Service2";
 				var calleeProcess = Process.Start(startInfo);
 				WriteLineToConsole("Spawned callee process");
 

src/Elastic.Apm.AspNetCore/WebRequestTransactionCreator.cs

@@ -14,6 +14,7 @@
 using Microsoft.AspNetCore.Http.Extensions;
 using Microsoft.AspNetCore.Http.Features;
 using Microsoft.AspNetCore.Routing;
+using Microsoft.Extensions.Primitives;
 
 namespace Elastic.Apm.AspNetCore
 {
@@ -35,23 +36,25 @@ internal static ITransaction StartTransactionAsync(HttpContext context, IApmLogg
 				ITransaction transaction;
 				var transactionName = $"{context.Request.Method} {context.Request.Path}";
 
-				if (context.Request.Headers.ContainsKey(TraceContext.TraceParentHeaderNamePrefixed)
-					|| context.Request.Headers.ContainsKey(TraceContext.TraceParentHeaderName))
-				{
-					var headerValue = context.Request.Headers.ContainsKey(TraceContext.TraceParentHeaderName)
-						? context.Request.Headers[TraceContext.TraceParentHeaderName].ToString()
-						: context.Request.Headers[TraceContext.TraceParentHeaderNamePrefixed].ToString();
+				var containsPrefixedTraceParentHeader =
+					context.Request.Headers.TryGetValue(TraceContext.TraceParentHeaderNamePrefixed, out var traceParentHeader);
+
+				var containsTraceParentHeader = false;
+				if (!containsPrefixedTraceParentHeader)
+					containsTraceParentHeader = context.Request.Headers.TryGetValue(TraceContext.TraceParentHeaderName, out traceParentHeader);
 
-					var tracingData = context.Request.Headers.ContainsKey(TraceContext.TraceStateHeaderName)
-						? TraceContext.TryExtractTracingData(headerValue, context.Request.Headers[TraceContext.TraceStateHeaderName].ToString())
-						: TraceContext.TryExtractTracingData(headerValue);
+				if (containsPrefixedTraceParentHeader || containsTraceParentHeader)
+				{
+					var tracingData = context.Request.Headers.TryGetValue(TraceContext.TraceStateHeaderName, out var traceStateHeader)
+						? TraceContext.TryExtractTracingData(traceParentHeader, traceStateHeader)
+						: TraceContext.TryExtractTracingData(traceParentHeader);
 
 					if (tracingData != null)
 					{
 						logger.Debug()
 							?.Log(
 								"Incoming request with {TraceParentHeaderName} header. DistributedTracingData: {DistributedTracingData}. Continuing trace.",
-								TraceContext.TraceParentHeaderNamePrefixed, tracingData);
+								containsPrefixedTraceParentHeader? TraceContext.TraceParentHeaderNamePrefixed : TraceContext.TraceParentHeaderName, tracingData);
 
 						transaction = tracer.StartTransaction(transactionName, ApiConstants.TypeRequest, tracingData);
 					}
@@ -60,7 +63,7 @@ internal static ITransaction StartTransactionAsync(HttpContext context, IApmLogg
 						logger.Debug()
 							?.Log(
 								"Incoming request with invalid {TraceParentHeaderName} header (received value: {TraceParentHeaderValue}). Starting trace with new trace id.",
-								TraceContext.TraceParentHeaderNamePrefixed, headerValue);
+								containsPrefixedTraceParentHeader? TraceContext.TraceParentHeaderNamePrefixed : TraceContext.TraceParentHeaderName, traceParentHeader);
 
 						transaction = tracer.StartTransaction(transactionName, ApiConstants.TypeRequest);
 					}

src/Elastic.Apm.AspNetFullFramework/ElasticApmModule.cs

@@ -15,6 +15,7 @@
 using Elastic.Apm.Helpers;
 using Elastic.Apm.Logging;
 using Elastic.Apm.Model;
+using TraceContext = Elastic.Apm.DistributedTracing.TraceContext;
 
 namespace Elastic.Apm.AspNetFullFramework
 {
@@ -133,7 +134,7 @@ private void ProcessBeginRequest(object sender)
 				_logger.Debug()
 					?.Log(
 						"Incoming request with {TraceParentHeaderName} header. DistributedTracingData: {DistributedTracingData} - continuing trace",
-						DistributedTracing.TraceContext.TraceParentHeaderNamePrefixed, distributedTracingData);
+						TraceContext.TraceParentHeaderName, distributedTracingData);
 
 				// we set ignoreActivity to true to avoid the HttpContext W3C DiagnosticSource issue (see https://github.com/elastic/apm-agent-dotnet/issues/867#issuecomment-650170150)
 				transaction = Agent.Instance.Tracer.StartTransaction(transactionName, ApiConstants.TypeRequest, distributedTracingData, true);
@@ -157,26 +158,23 @@ private void ProcessBeginRequest(object sender)
 		/// <returns>Null if traceparent is not set, otherwise the filled DistributedTracingData instance</returns>
 		private DistributedTracingData ExtractIncomingDistributedTracingData(HttpRequest request)
 		{
-			var traceParentHeaderValue = request.Unvalidated.Headers.Get(DistributedTracing.TraceContext.TraceParentHeaderName);
+			var traceParentHeaderValue = request.Unvalidated.Headers.Get(TraceContext.TraceParentHeaderName);
 			// ReSharper disable once InvertIf
 			if (traceParentHeaderValue == null)
 			{
-				traceParentHeaderValue = request.Unvalidated.Headers.Get(DistributedTracing.TraceContext.TraceParentHeaderNamePrefixed);
+				traceParentHeaderValue = request.Unvalidated.Headers.Get(TraceContext.TraceParentHeaderNamePrefixed);
 
 				if (traceParentHeaderValue == null)
 				{
 					_logger.Debug()
 						?.Log("Incoming request doesn't have {TraceParentHeaderName} header - " +
-							"it means request doesn't have incoming distributed tracing data", DistributedTracing.TraceContext.TraceParentHeaderNamePrefixed);
+							"it means request doesn't have incoming distributed tracing data", TraceContext.TraceParentHeaderNamePrefixed);
 					return null;
 				}
 			}
 
-			var traceStateHeaderValue = request.Unvalidated.Headers.Get(DistributedTracing.TraceContext.TraceStateHeaderName);
-
-			return traceStateHeaderValue != null
-				? DistributedTracing.TraceContext.TryExtractTracingData(traceParentHeaderValue, traceStateHeaderValue)
-				: DistributedTracing.TraceContext.TryExtractTracingData(traceParentHeaderValue);
+			var traceStateHeaderValue = request.Unvalidated.Headers.Get(TraceContext.TraceStateHeaderName);
+			return TraceContext.TryExtractTracingData(traceParentHeaderValue, traceStateHeaderValue);
 		}
 
 		private static void FillSampledTransactionContextRequest(HttpRequest request, ITransaction transaction)

src/Elastic.Apm/Api/DistributedTracingData.cs

@@ -2,21 +2,24 @@
 // Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
 // See the LICENSE file in the project root for more information
 
+using System;
 using Elastic.Apm.DistributedTracing;
 using Elastic.Apm.Helpers;
 
 namespace Elastic.Apm.Api
 {
 	/// <summary>
-	/// An object encapsulating data passed from the caller to the callee in distributed tracing in order to correlate between
-	/// them.
-	/// Its purpose is similar to that of "traceparent" header described at https://www.w3.org/TR/trace-context/
-	/// See samples/ApiSamples/Program.cs for an example on how to manually pass distributed tracing data between the caller
-	/// and the callee.
+	/// Encapsulates distributed tracing data passed from the caller to the callee in order to correlate calls between them.
+	/// Its purpose is similar to that of traceparent and tracestate headers described in the
+	/// <a href="https://www.w3.org/TR/trace-context/">Trace Context specification</a>
 	/// </summary>
+	/// <example>
+	/// See sample/ApiSamples/Program.cs for an example on how to manually pass distributed tracing data between the caller
+	/// and the callee
+	/// </example>
 	public class DistributedTracingData
 	{
-		internal DistributedTracingData(string traceId, string parentId, bool flagRecorded, string traceState = null)
+		internal DistributedTracingData(string traceId, string parentId, bool flagRecorded, TraceState traceState = null)
 		{
 			TraceId = traceId;
 			ParentId = parentId;
@@ -25,11 +28,10 @@ internal DistributedTracingData(string traceId, string parentId, bool flagRecord
 		}
 
 		internal bool FlagRecorded { get; }
-
-		internal bool HasTraceState => !string.IsNullOrEmpty(TraceState);
+		internal bool HasTraceState => TraceState != null;
 		internal string ParentId { get; }
 		internal string TraceId { get; }
-		internal string TraceState { get; }
+		internal TraceState TraceState { get; }
 
 		/// <summary>
 		/// Serializes this instance to a string.
@@ -40,8 +42,25 @@ internal DistributedTracingData(string traceId, string parentId, bool flagRecord
 		/// <returns>
 		/// String containing the instance in serialized form.
 		/// </returns>
+		[Obsolete("Use " + nameof(TraceparentTextHeader))]
 		public string SerializeToString() => TraceContext.BuildTraceparent(this);
 
+		/// <summary>
+		/// Gets the traceparent text header from this instance of <see cref="DistributedTracingData"/>.
+		/// The traceparent header should be passed to the callee.
+		/// <see cref="FromTraceContext" /> should be used to instantiate <see cref="DistributedTracingData"/> at the callee side.
+		/// </summary>
+		/// <returns>A new instance of the traceparent header</returns>
+		public string TraceparentTextHeader => TraceContext.BuildTraceparent(this);
+
+		/// <summary>
+		/// Gets the tracestate text header from this instance of <see cref="DistributedTracingData"/>.
+		/// The tracestate header should be passed to the callee.
+		/// <see cref="FromTraceContext" /> should be used to instantiate <see cref="DistributedTracingData"/> at the callee side.
+		/// </summary>
+		/// <returns>A new instance of the tracestate header</returns>
+		public string TracestateTextHeader => TraceState?.ToTextHeader();
+
 		/// <summary>
 		/// Deserializes an instance from a string.
 		/// This method should be used at the callee side and the deserialized instance can be passed to
@@ -51,11 +70,25 @@ internal DistributedTracingData(string traceId, string parentId, bool flagRecord
 		/// <returns>
 		/// Instance deserialized from <paramref name="serialized" />.
 		/// </returns>
+		[Obsolete("Use " + nameof(FromTraceContext))]
 		public static DistributedTracingData TryDeserializeFromString(string serialized) => TraceContext.TryExtractTracingData(serialized);
 
+		/// <summary>
+		/// Creates an instance of <see cref="DistributedTracingData"/> from Trace Context headers
+		/// </summary>
+		/// <param name="traceParent">The traceparent header value</param>
+		/// <param name="traceState">The tracestate header value. If there are multiple header values, join as comma-separated</param>
+		/// <returns>A new instance of <see cref="DistributedTracingData"/> if the header values represent a valid trace context, otherwise null
+		/// </returns>
+		public static DistributedTracingData FromTraceContext(string traceParent, string traceState = null) =>
+			TraceContext.TryExtractTracingData(traceParent, traceState);
+
 		public override string ToString() => new ToStringBuilder(nameof(DistributedTracingData))
 		{
-			{ "TraceId", TraceId }, { "ParentId", ParentId }, { "FlagRecorded", FlagRecorded }
+			{ nameof(TraceId), TraceId },
+			{ nameof(ParentId), ParentId },
+			{ nameof(FlagRecorded), FlagRecorded },
+			{ nameof(TraceState), TracestateTextHeader }
 		}.ToString();
 	}
 }

src/Elastic.Apm/Config/IConfigurationReader.cs

@@ -190,6 +190,11 @@ public interface IConfigurationReader
 		/// These apply for example to HTTP headers and application/x-www-form-urlencoded data.
 		/// </summary>
 		IReadOnlyList<WildcardMatcher> SanitizeFieldNames { get; }
+
+		/// <summary>
+		/// A secret token to ensure that only your agents can send data to your APM server.
+		/// Both agents and APM server have to be configured with the same secret token.
+		/// </summary>
 		string SecretToken { get; }
 
 		/// <summary>
@@ -199,7 +204,7 @@ public interface IConfigurationReader
 		IReadOnlyList<Uri> ServerUrls { get; }
 
 		/// <summary>
-		/// The URL for APM server
+		/// The URL for APM server.
 		/// </summary>
 		Uri ServerUrl { get; }
 
@@ -262,12 +267,17 @@ public interface IConfigurationReader
 		/// </summary>
 		int TransactionMaxSpans { get; }
 
+		/// <summary>
+		/// The sample rate for transactions.
+		/// By default, the agent will sample every transaction (e.g. a request to your service). To reduce overhead and storage requirements,
+		/// the sample rate can be set to a value between 0.0 and 1.0. The agent still records the overall time and result for unsampled
+		/// transactions, but no context information, labels, or spans are recorded.
+		/// </summary>
 		double TransactionSampleRate { get; }
 
 		/// <summary>
-		/// If true, for all outgoing HTTP requests the agent stores the traceparent in a header prefixed with elastic-apm
-		/// (elastic-apm-traceparent)
-		/// otherwise it'll use the official header name from w3c, which is "traceparewnt".
+		/// If <c>true</c>, for all outgoing HTTP requests the agent stores the traceparent in a "elastic-apm-traceparent" header name.
+		/// Otherwise, it'll use the official w3c "traceparent" header name.
 		/// </summary>
 		bool UseElasticTraceparentHeader { get; }
 

src/Elastic.Apm/DiagnosticListeners/HttpDiagnosticListenerCoreImpl.cs

@@ -3,6 +3,7 @@
 // See the LICENSE file in the project root for more information
 
 using System;
+using System.Collections.Generic;
 using System.Net.Http;
 
 namespace Elastic.Apm.DiagnosticListeners
@@ -25,6 +26,15 @@ public HttpDiagnosticListenerCoreImpl(IApmAgent agent)
 		protected override bool RequestHeadersContain(HttpRequestMessage request, string headerName)
 			=> request.Headers.Contains(headerName);
 
+		protected override bool RequestTryGetHeader(HttpRequestMessage request, string headerName, out string value)
+		{
+			var contains = request.Headers.TryGetValues(headerName, out var values);
+			value = contains
+				? string.Join(",", values)
+				: null;
+			return contains;
+		}
+
 		protected override void RequestHeadersAdd(HttpRequestMessage request, string headerName, string headerValue)
 		{
 			if(!string.IsNullOrEmpty(headerValue))

src/Elastic.Apm/DiagnosticListeners/HttpDiagnosticListenerFullFrameworkImpl.cs

@@ -3,6 +3,7 @@
 // See the LICENSE file in the project root for more information
 
 using System;
+using System.Collections.Generic;
 using System.Net;
 using System.Reflection;
 using Elastic.Apm.Logging;
@@ -24,11 +25,8 @@ public HttpDiagnosticListenerFullFrameworkImpl(IApmAgent agent)
 
 		protected override string RequestGetMethod(HttpWebRequest request) => request.Method;
 
-		protected override bool RequestHeadersContain(HttpWebRequest request, string headerName)
-		{
-			var values = request.Headers.GetValues(headerName);
-			return values != null && values.Length > 0;
-		}
+		protected override bool RequestHeadersContain(HttpWebRequest request, string headerName) =>
+			RequestTryGetHeader(request, headerName, out _);
 
 		protected override void RequestHeadersAdd(HttpWebRequest request, string headerName, string headerValue)
 		{
@@ -38,6 +36,12 @@ protected override void RequestHeadersAdd(HttpWebRequest request, string headerN
 
 		protected override int ResponseGetStatusCode(HttpWebResponse response) => (int)response.StatusCode;
 
+		protected override bool RequestTryGetHeader(HttpWebRequest request, string headerName, out string value)
+		{
+			value = request.Headers.Get(headerName);
+			return value != null;
+		}
+
 		/// <summary>
 		/// In Full Framework "System.Net.Http.Desktop.HttpRequestOut.Ex.Stop" does not send the exception property.
 		/// Therefore we have a specialized ProcessExceptionEvent for Full Framework.