From 90d93fb637d65b7f5a002cef0216e84b1cf73aca Mon Sep 17 00:00:00 2001 From: Cijo Thomas Date: Thu, 30 Sep 2021 13:49:52 -0700 Subject: [PATCH 01/39] Support View Part2 - specify custom buckets for Histogram (#2432) --- .../ConsoleMetricExporter.cs | 60 +++++++++-------- .../Implementation/MetricItemExtensions.cs | 11 ++-- .../PrometheusExporterExtensions.cs | 25 +++---- src/OpenTelemetry/Metrics/AggregationType.cs | 5 ++ src/OpenTelemetry/Metrics/AggregatorStore.cs | 8 ++- src/OpenTelemetry/Metrics/MeterProviderSdk.cs | 15 ++++- src/OpenTelemetry/Metrics/Metric.cs | 17 ++++- src/OpenTelemetry/Metrics/MetricPoint.cs | 40 +++++++++++- .../Metrics/AggregatorTest.cs | 59 ++++++++++++++++- .../Metrics/MetricViewTests.cs | 65 +++++++++++++++++++ 10 files changed, 251 insertions(+), 54 deletions(-) diff --git a/src/OpenTelemetry.Exporter.Console/ConsoleMetricExporter.cs b/src/OpenTelemetry.Exporter.Console/ConsoleMetricExporter.cs index 3b8169dc237..49092a3187d 100644 --- a/src/OpenTelemetry.Exporter.Console/ConsoleMetricExporter.cs +++ b/src/OpenTelemetry.Exporter.Console/ConsoleMetricExporter.cs @@ -99,37 +99,41 @@ public override ExportResult Export(in Batch batch) { var bucketsBuilder = new StringBuilder(); bucketsBuilder.Append($"Sum: {metricPoint.DoubleValue} Count: {metricPoint.LongValue} \n"); - for (int i = 0; i < metricPoint.ExplicitBounds.Length + 1; i++) + + if (metricPoint.ExplicitBounds != null) { - if (i == 0) - { - bucketsBuilder.Append("(-Infinity,"); - bucketsBuilder.Append(metricPoint.ExplicitBounds[i]); - bucketsBuilder.Append(']'); - bucketsBuilder.Append(':'); - bucketsBuilder.Append(metricPoint.BucketCounts[i]); - } - else if (i == metricPoint.ExplicitBounds.Length) + for (int i = 0; i < metricPoint.ExplicitBounds.Length + 1; i++) { - bucketsBuilder.Append('('); - bucketsBuilder.Append(metricPoint.ExplicitBounds[i - 1]); - bucketsBuilder.Append(','); - bucketsBuilder.Append("+Infinity]"); - bucketsBuilder.Append(':'); - bucketsBuilder.Append(metricPoint.BucketCounts[i]); + if (i == 0) + { + bucketsBuilder.Append("(-Infinity,"); + bucketsBuilder.Append(metricPoint.ExplicitBounds[i]); + bucketsBuilder.Append(']'); + bucketsBuilder.Append(':'); + bucketsBuilder.Append(metricPoint.BucketCounts[i]); + } + else if (i == metricPoint.ExplicitBounds.Length) + { + bucketsBuilder.Append('('); + bucketsBuilder.Append(metricPoint.ExplicitBounds[i - 1]); + bucketsBuilder.Append(','); + bucketsBuilder.Append("+Infinity]"); + bucketsBuilder.Append(':'); + bucketsBuilder.Append(metricPoint.BucketCounts[i]); + } + else + { + bucketsBuilder.Append('('); + bucketsBuilder.Append(metricPoint.ExplicitBounds[i - 1]); + bucketsBuilder.Append(','); + bucketsBuilder.Append(metricPoint.ExplicitBounds[i]); + bucketsBuilder.Append(']'); + bucketsBuilder.Append(':'); + bucketsBuilder.Append(metricPoint.BucketCounts[i]); + } + + bucketsBuilder.AppendLine(); } - else - { - bucketsBuilder.Append('('); - bucketsBuilder.Append(metricPoint.ExplicitBounds[i - 1]); - bucketsBuilder.Append(','); - bucketsBuilder.Append(metricPoint.ExplicitBounds[i]); - bucketsBuilder.Append(']'); - bucketsBuilder.Append(':'); - bucketsBuilder.Append(metricPoint.BucketCounts[i]); - } - - bucketsBuilder.AppendLine(); } valueDisplay = bucketsBuilder.ToString(); diff --git a/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/Implementation/MetricItemExtensions.cs b/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/Implementation/MetricItemExtensions.cs index 1e935fe5500..dbc704bf208 100644 --- a/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/Implementation/MetricItemExtensions.cs +++ b/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/Implementation/MetricItemExtensions.cs @@ -250,12 +250,15 @@ internal static OtlpMetrics.Metric ToOtlpMetric(this Metric metric) dataPoint.Count = (ulong)metricPoint.LongValue; dataPoint.Sum = metricPoint.DoubleValue; - for (int i = 0; i < metricPoint.BucketCounts.Length; i++) + if (metricPoint.BucketCounts != null) { - dataPoint.BucketCounts.Add((ulong)metricPoint.BucketCounts[i]); - if (i < metricPoint.BucketCounts.Length - 1) + for (int i = 0; i < metricPoint.BucketCounts.Length; i++) { - dataPoint.ExplicitBounds.Add(metricPoint.ExplicitBounds[i]); + dataPoint.BucketCounts.Add((ulong)metricPoint.BucketCounts[i]); + if (i < metricPoint.BucketCounts.Length - 1) + { + dataPoint.ExplicitBounds.Add(metricPoint.ExplicitBounds[i]); + } } } diff --git a/src/OpenTelemetry.Exporter.Prometheus/Implementation/PrometheusExporterExtensions.cs b/src/OpenTelemetry.Exporter.Prometheus/Implementation/PrometheusExporterExtensions.cs index 88e504fa32c..74d5c4345a9 100644 --- a/src/OpenTelemetry.Exporter.Prometheus/Implementation/PrometheusExporterExtensions.cs +++ b/src/OpenTelemetry.Exporter.Prometheus/Implementation/PrometheusExporterExtensions.cs @@ -164,18 +164,21 @@ private static void WriteHistogramMetrics(Metric metric, PrometheusMetricBuilder metricValueBuilderCount = metricValueBuilderCount.WithValue(metricPoint.LongValue); metricValueBuilderCount.AddLabels(metricPoint.Keys, metricPoint.Values); - long totalCount = 0; - for (int i = 0; i < metricPoint.ExplicitBounds.Length + 1; i++) + if (metricPoint.ExplicitBounds != null) { - totalCount += metricPoint.BucketCounts[i]; - var metricValueBuilderBuckets = builder.AddValue(); - metricValueBuilderBuckets.WithName(metric.Name + PrometheusHistogramBucketPostFix); - metricValueBuilderBuckets = metricValueBuilderBuckets.WithValue(totalCount); - metricValueBuilderBuckets.AddLabels(metricPoint.Keys, metricPoint.Values); - - var bucketName = i == metricPoint.ExplicitBounds.Length ? - PrometheusHistogramBucketLabelPositiveInfinity : metricPoint.ExplicitBounds[i].ToString(CultureInfo.InvariantCulture); - metricValueBuilderBuckets.WithLabel(PrometheusHistogramBucketLabelLessThan, bucketName); + long totalCount = 0; + for (int i = 0; i < metricPoint.ExplicitBounds.Length + 1; i++) + { + totalCount += metricPoint.BucketCounts[i]; + var metricValueBuilderBuckets = builder.AddValue(); + metricValueBuilderBuckets.WithName(metric.Name + PrometheusHistogramBucketPostFix); + metricValueBuilderBuckets = metricValueBuilderBuckets.WithValue(totalCount); + metricValueBuilderBuckets.AddLabels(metricPoint.Keys, metricPoint.Values); + + var bucketName = i == metricPoint.ExplicitBounds.Length ? + PrometheusHistogramBucketLabelPositiveInfinity : metricPoint.ExplicitBounds[i].ToString(CultureInfo.InvariantCulture); + metricValueBuilderBuckets.WithLabel(PrometheusHistogramBucketLabelLessThan, bucketName); + } } } } diff --git a/src/OpenTelemetry/Metrics/AggregationType.cs b/src/OpenTelemetry/Metrics/AggregationType.cs index 29b9667215a..bd87c9e5b4f 100644 --- a/src/OpenTelemetry/Metrics/AggregationType.cs +++ b/src/OpenTelemetry/Metrics/AggregationType.cs @@ -57,5 +57,10 @@ internal enum AggregationType /// Histogram. /// Histogram = 6, + + /// + /// Histogram with sum, count only. + /// + HistogramSumCount = 7, } } diff --git a/src/OpenTelemetry/Metrics/AggregatorStore.cs b/src/OpenTelemetry/Metrics/AggregatorStore.cs index 88f60b4fc66..0d99cf61442 100644 --- a/src/OpenTelemetry/Metrics/AggregatorStore.cs +++ b/src/OpenTelemetry/Metrics/AggregatorStore.cs @@ -36,14 +36,16 @@ internal class AggregatorStore private int metricPointIndex = 0; private bool zeroTagMetricPointInitialized; private AggregationType aggType; + private double[] histogramBounds; private DateTimeOffset startTimeExclusive; private DateTimeOffset endTimeInclusive; - internal AggregatorStore(AggregationType aggType, AggregationTemporality temporality) + internal AggregatorStore(AggregationType aggType, AggregationTemporality temporality, double[] histogramBounds) { this.metrics = new MetricPoint[MaxMetricPoints]; this.aggType = aggType; this.temporality = temporality; + this.histogramBounds = histogramBounds; this.startTimeExclusive = DateTimeOffset.UtcNow; } @@ -59,7 +61,7 @@ internal int FindMetricAggregators(ReadOnlySpan> ta if (!this.zeroTagMetricPointInitialized) { var dt = DateTimeOffset.UtcNow; - this.metrics[0] = new MetricPoint(this.aggType, dt, null, null); + this.metrics[0] = new MetricPoint(this.aggType, dt, null, null, this.histogramBounds); this.zeroTagMetricPointInitialized = true; } } @@ -137,7 +139,7 @@ internal int FindMetricAggregators(ReadOnlySpan> ta ref var metricPoint = ref this.metrics[aggregatorIndex]; var dt = DateTimeOffset.UtcNow; - metricPoint = new MetricPoint(this.aggType, dt, seqKey, seqVal); + metricPoint = new MetricPoint(this.aggType, dt, seqKey, seqVal, this.histogramBounds); // Add to dictionary *after* initializing MetricPoint // as other threads can start writing to the diff --git a/src/OpenTelemetry/Metrics/MeterProviderSdk.cs b/src/OpenTelemetry/Metrics/MeterProviderSdk.cs index 8ae8271d5d3..81be9ae72d9 100644 --- a/src/OpenTelemetry/Metrics/MeterProviderSdk.cs +++ b/src/OpenTelemetry/Metrics/MeterProviderSdk.cs @@ -134,7 +134,8 @@ internal MeterProviderSdk( { for (int i = 0; i < maxCountMetricsToBeCreated; i++) { - var metricStreamName = metricStreamConfigs[i]?.Name ?? instrument.Name; + var metricStreamConfig = metricStreamConfigs[i]; + var metricStreamName = metricStreamConfig?.Name ?? instrument.Name; if (this.metricStreamNames.ContainsKey(metricStreamName)) { // TODO: Log that instrument is ignored @@ -151,7 +152,17 @@ internal MeterProviderSdk( } else { - var metric = new Metric(instrument, temporality, metricStreamName); + Metric metric; + if (metricStreamConfig is HistogramConfiguration histogramConfig + && histogramConfig.BucketBounds != null) + { + metric = new Metric(instrument, temporality, histogramConfig.BucketBounds, metricStreamName); + } + else + { + metric = new Metric(instrument, temporality, metricStreamName); + } + this.metrics[index] = metric; metrics.Add(metric); this.metricStreamNames.Add(metricStreamName, true); diff --git a/src/OpenTelemetry/Metrics/Metric.cs b/src/OpenTelemetry/Metrics/Metric.cs index 08da95c7f78..ca3ebff5375 100644 --- a/src/OpenTelemetry/Metrics/Metric.cs +++ b/src/OpenTelemetry/Metrics/Metric.cs @@ -22,9 +22,15 @@ namespace OpenTelemetry.Metrics { public class Metric { + internal static readonly double[] DefaultHistogramBounds = new double[] { 0, 5, 10, 25, 50, 75, 100, 250, 500, 1000 }; private AggregatorStore aggStore; internal Metric(Instrument instrument, AggregationTemporality temporality, string metricName = "") + : this(instrument, temporality, DefaultHistogramBounds, metricName) + { + } + + internal Metric(Instrument instrument, AggregationTemporality temporality, double[] histogramBounds, string metricName = "") { this.Name = string.IsNullOrWhiteSpace(metricName) ? instrument.Name : metricName; this.Description = instrument.Description; @@ -80,15 +86,22 @@ internal Metric(Instrument instrument, AggregationTemporality temporality, strin || instrument.GetType() == typeof(Histogram) || instrument.GetType() == typeof(Histogram)) { - aggType = AggregationType.Histogram; this.MetricType = MetricType.Histogram; + if (histogramBounds.Length == 0) + { + aggType = AggregationType.HistogramSumCount; + } + else + { + aggType = AggregationType.Histogram; + } } else { // TODO: Log and assign some invalid Enum. } - this.aggStore = new AggregatorStore(aggType, temporality); + this.aggStore = new AggregatorStore(aggType, temporality, histogramBounds); this.Temporality = temporality; } diff --git a/src/OpenTelemetry/Metrics/MetricPoint.cs b/src/OpenTelemetry/Metrics/MetricPoint.cs index 1750499bf2e..b4eb717eaf4 100644 --- a/src/OpenTelemetry/Metrics/MetricPoint.cs +++ b/src/OpenTelemetry/Metrics/MetricPoint.cs @@ -32,7 +32,8 @@ internal MetricPoint( AggregationType aggType, DateTimeOffset startTime, string[] keys, - object[] values) + object[] values, + double[] histogramBounds) { this.AggType = aggType; this.StartTime = startTime; @@ -48,11 +49,18 @@ internal MetricPoint( if (this.AggType == AggregationType.Histogram) { - this.ExplicitBounds = new double[] { 0, 5, 10, 25, 50, 75, 100, 250, 500, 1000 }; + this.ExplicitBounds = histogramBounds; this.BucketCounts = new long[this.ExplicitBounds.Length + 1]; this.bucketCounts = new long[this.ExplicitBounds.Length + 1]; this.lockObject = new object(); } + else if (this.AggType == AggregationType.HistogramSumCount) + { + this.ExplicitBounds = null; + this.BucketCounts = null; + this.bucketCounts = null; + this.lockObject = new object(); + } else { this.ExplicitBounds = null; @@ -103,6 +111,7 @@ internal void Update(long number) } case AggregationType.Histogram: + case AggregationType.HistogramSumCount: { this.Update((double)number); break; @@ -157,6 +166,17 @@ internal void Update(double number) this.bucketCounts[i]++; } + break; + } + + case AggregationType.HistogramSumCount: + { + lock (this.lockObject) + { + this.longVal++; + this.doubleVal += number; + } + break; } } @@ -249,6 +269,22 @@ internal void TakeSnapShot(bool outputDelta) } } + break; + } + + case AggregationType.HistogramSumCount: + { + lock (this.lockObject) + { + this.LongValue = this.longVal; + this.DoubleValue = this.doubleVal; + if (outputDelta) + { + this.longVal = 0; + this.doubleVal = 0; + } + } + break; } } diff --git a/test/OpenTelemetry.Tests/Metrics/AggregatorTest.cs b/test/OpenTelemetry.Tests/Metrics/AggregatorTest.cs index 5d615b60311..002013dab8e 100644 --- a/test/OpenTelemetry.Tests/Metrics/AggregatorTest.cs +++ b/test/OpenTelemetry.Tests/Metrics/AggregatorTest.cs @@ -22,9 +22,9 @@ namespace OpenTelemetry.Metrics.Tests public class AggregatorTest { [Fact] - public void HistogramDistributeToAllBuckets() + public void HistogramDistributeToAllBucketsDefault() { - var histogramPoint = new MetricPoint(AggregationType.Histogram, DateTimeOffset.Now, null, null); + var histogramPoint = new MetricPoint(AggregationType.Histogram, DateTimeOffset.Now, null, null, Metric.DefaultHistogramBounds); histogramPoint.Update(-1); histogramPoint.Update(0); histogramPoint.Update(2); @@ -55,5 +55,60 @@ public void HistogramDistributeToAllBuckets() Assert.Equal(2, histogramPoint.BucketCounts[i]); } } + + [Fact] + public void HistogramDistributeToAllBucketsCustom() + { + var bounds = new double[] { 10, 20 }; + var histogramPoint = new MetricPoint(AggregationType.Histogram, DateTimeOffset.Now, null, null, bounds); + + // 5 recordings <=10 + histogramPoint.Update(-10); + histogramPoint.Update(0); + histogramPoint.Update(1); + histogramPoint.Update(9); + histogramPoint.Update(10); + + // 2 recordings >10, <=20 + histogramPoint.Update(11); + histogramPoint.Update(19); + + histogramPoint.TakeSnapShot(true); + + // Sum of all recordings + Assert.Equal(40, histogramPoint.DoubleValue); + + // Count = # of recordings + Assert.Equal(7, histogramPoint.LongValue); + Assert.Equal(bounds.Length + 1, histogramPoint.BucketCounts.Length); + Assert.Equal(5, histogramPoint.BucketCounts[0]); + Assert.Equal(2, histogramPoint.BucketCounts[1]); + Assert.Equal(0, histogramPoint.BucketCounts[2]); + } + + [Fact] + public void HistogramWithOnlySumCount() + { + var bounds = new double[] { }; + var histogramPoint = new MetricPoint(AggregationType.HistogramSumCount, DateTimeOffset.Now, null, null, bounds); + + histogramPoint.Update(-10); + histogramPoint.Update(0); + histogramPoint.Update(1); + histogramPoint.Update(9); + histogramPoint.Update(10); + histogramPoint.Update(11); + histogramPoint.Update(19); + + histogramPoint.TakeSnapShot(true); + + // Sum of all recordings + Assert.Equal(40, histogramPoint.DoubleValue); + + // Count = # of recordings + Assert.Equal(7, histogramPoint.LongValue); + Assert.Null(histogramPoint.BucketCounts); + Assert.Null(histogramPoint.ExplicitBounds); + } } } diff --git a/test/OpenTelemetry.Tests/Metrics/MetricViewTests.cs b/test/OpenTelemetry.Tests/Metrics/MetricViewTests.cs index b30bc1a3791..2e6313d871a 100644 --- a/test/OpenTelemetry.Tests/Metrics/MetricViewTests.cs +++ b/test/OpenTelemetry.Tests/Metrics/MetricViewTests.cs @@ -162,5 +162,70 @@ public void ViewToProduceMultipleStreamsWithDuplicatesFromInstrument() Assert.Equal("renamedStream1", exportedItems[0].Name); Assert.Equal("renamedStream2", exportedItems[1].Name); } + + [Fact] + public void ViewToProduceCustomHistogramBound() + { + using var meter1 = new Meter("ViewToProduceCustomHistogramBoundTest"); + var exportedItems = new List(); + var bounds = new double[] { 10, 20 }; + using var meterProvider = Sdk.CreateMeterProviderBuilder() + .AddSource(meter1.Name) + .AddView("MyHistogram", new HistogramConfiguration() { Name = "MyHistogramDefaultBound" }) + .AddView("MyHistogram", new HistogramConfiguration() { BucketBounds = bounds }) + .AddInMemoryExporter(exportedItems) + .Build(); + + var histogram = meter1.CreateHistogram("MyHistogram"); + histogram.Record(-10); + histogram.Record(0); + histogram.Record(1); + histogram.Record(9); + histogram.Record(10); + histogram.Record(11); + histogram.Record(19); + meterProvider.ForceFlush(MaxTimeToAllowForFlush); + Assert.Equal(2, exportedItems.Count); + var metricDefault = exportedItems[0]; + var metricCustom = exportedItems[1]; + + Assert.Equal("MyHistogramDefaultBound", metricDefault.Name); + Assert.Equal("MyHistogram", metricCustom.Name); + + List metricPointsDefault = new List(); + foreach (ref var mp in metricDefault.GetMetricPoints()) + { + metricPointsDefault.Add(mp); + } + + Assert.Single(metricPointsDefault); + var histogramPoint = metricPointsDefault[0]; + + Assert.Equal(40, histogramPoint.DoubleValue); + Assert.Equal(7, histogramPoint.LongValue); + Assert.Equal(Metric.DefaultHistogramBounds.Length + 1, histogramPoint.BucketCounts.Length); + Assert.Equal(2, histogramPoint.BucketCounts[0]); + Assert.Equal(1, histogramPoint.BucketCounts[1]); + Assert.Equal(2, histogramPoint.BucketCounts[2]); + Assert.Equal(2, histogramPoint.BucketCounts[3]); + Assert.Equal(0, histogramPoint.BucketCounts[4]); + Assert.Equal(0, histogramPoint.BucketCounts[5]); + + List metricPointsCustom = new List(); + foreach (ref var mp in metricCustom.GetMetricPoints()) + { + metricPointsCustom.Add(mp); + } + + Assert.Single(metricPointsCustom); + histogramPoint = metricPointsCustom[0]; + + Assert.Equal(40, histogramPoint.DoubleValue); + Assert.Equal(7, histogramPoint.LongValue); + Assert.Equal(bounds.Length + 1, histogramPoint.BucketCounts.Length); + Assert.Equal(5, histogramPoint.BucketCounts[0]); + Assert.Equal(2, histogramPoint.BucketCounts[1]); + Assert.Equal(0, histogramPoint.BucketCounts[2]); + } } } From 124c12afdfd0bbaad02c215b6b07e0bc7804b2d0 Mon Sep 17 00:00:00 2001 From: Cijo Thomas Date: Thu, 30 Sep 2021 14:41:29 -0700 Subject: [PATCH 02/39] Prometheus fixes (#2433) --- examples/Console/Program.cs | 2 +- examples/Console/TestPrometheusExporter.cs | 2 +- .../Implementation/PrometheusExporterMetricsHttpServer.cs | 5 +++++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/examples/Console/Program.cs b/examples/Console/Program.cs index 81572e68317..dbba17e57f8 100644 --- a/examples/Console/Program.cs +++ b/examples/Console/Program.cs @@ -31,7 +31,7 @@ public class Program /// dotnet run -p Examples.Console.csproj inmemory /// dotnet run -p Examples.Console.csproj zipkin -u http://localhost:9411/api/v2/spans /// dotnet run -p Examples.Console.csproj jaeger -h localhost -p 6831 - /// dotnet run -p Examples.Console.csproj prometheus -i 15 -p 9184 -d 2 + /// dotnet run -p Examples.Console.csproj prometheus -p 9184 -d 2 /// dotnet run -p Examples.Console.csproj otlp -e "http://localhost:4317" /// dotnet run -p Examples.Console.csproj zpages /// dotnet run -p Examples.Console.csproj metrics --help diff --git a/examples/Console/TestPrometheusExporter.cs b/examples/Console/TestPrometheusExporter.cs index a1f66fd0ae5..565dc5c8f4a 100644 --- a/examples/Console/TestPrometheusExporter.cs +++ b/examples/Console/TestPrometheusExporter.cs @@ -52,7 +52,7 @@ internal static object Run(int port, int totalDurationInMins) .AddPrometheusExporter(opt => { opt.StartHttpListener = true; - opt.HttpListenerPrefixes = new string[] { $"http://*:{port}/" }; + opt.HttpListenerPrefixes = new string[] { $"http://localhost:{port}/" }; }) .Build(); diff --git a/src/OpenTelemetry.Exporter.Prometheus/Implementation/PrometheusExporterMetricsHttpServer.cs b/src/OpenTelemetry.Exporter.Prometheus/Implementation/PrometheusExporterMetricsHttpServer.cs index 909f1ff2ebb..7db001afe40 100644 --- a/src/OpenTelemetry.Exporter.Prometheus/Implementation/PrometheusExporterMetricsHttpServer.cs +++ b/src/OpenTelemetry.Exporter.Prometheus/Implementation/PrometheusExporterMetricsHttpServer.cs @@ -53,6 +53,11 @@ public PrometheusExporterMetricsHttpServer(PrometheusExporter exporter) path = $"/{path}"; } + if (!path.EndsWith("/")) + { + path = $"{path}/"; + } + foreach (string prefix in exporter.Options.HttpListenerPrefixes) { this.httpListener.Prefixes.Add($"{prefix.TrimEnd('/')}{path}"); From c0af496fd8ff4f6b60cfe6353cbec81052519f72 Mon Sep 17 00:00:00 2001 From: Tom Tan Date: Thu, 30 Sep 2021 16:20:10 -0700 Subject: [PATCH 03/39] Add warning when running stress test with DEBUG build (#2436) * Add warning when running stress test with DEBUG build * Revert unexpected change --- test/OpenTelemetry.Tests.Stress/Skeleton.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/OpenTelemetry.Tests.Stress/Skeleton.cs b/test/OpenTelemetry.Tests.Stress/Skeleton.cs index 5374b78e4cf..5033abf903a 100644 --- a/test/OpenTelemetry.Tests.Stress/Skeleton.cs +++ b/test/OpenTelemetry.Tests.Stress/Skeleton.cs @@ -30,6 +30,10 @@ public partial class Program public static void Stress(int concurrency = 0) { +#if DEBUG + Console.WriteLine("***WARNING*** The current build is DEBUG which may affect timing!\n"); +#endif + if (concurrency < 0) { throw new ArgumentOutOfRangeException(nameof(concurrency), "concurrency level should be a non-negative number."); From 8a8dd9716b28ee96d36dcce947149eed797e73a0 Mon Sep 17 00:00:00 2001 From: Alan West <3676547+alanwest@users.noreply.github.com> Date: Thu, 30 Sep 2021 16:32:25 -0700 Subject: [PATCH 04/39] Remove InternalsVisibleTo for OTLP exporter project (#2434) --- .../OpenTelemetry.Exporter.OpenTelemetryProtocol.csproj | 8 ++------ src/OpenTelemetry/AssemblyInfo.cs | 4 ---- .../OtlpMetricsExporterTests.cs | 8 +------- .../OtlpTraceExporterTests.cs | 8 +------- 4 files changed, 4 insertions(+), 24 deletions(-) diff --git a/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/OpenTelemetry.Exporter.OpenTelemetryProtocol.csproj b/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/OpenTelemetry.Exporter.OpenTelemetryProtocol.csproj index 72de9c3ee33..ecee7ccec98 100644 --- a/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/OpenTelemetry.Exporter.OpenTelemetryProtocol.csproj +++ b/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/OpenTelemetry.Exporter.OpenTelemetryProtocol.csproj @@ -32,14 +32,10 @@ - - - + diff --git a/src/OpenTelemetry/AssemblyInfo.cs b/src/OpenTelemetry/AssemblyInfo.cs index d6dd47bc2c9..8d6f5f821b6 100644 --- a/src/OpenTelemetry/AssemblyInfo.cs +++ b/src/OpenTelemetry/AssemblyInfo.cs @@ -20,7 +20,3 @@ [assembly: InternalsVisibleTo("OpenTelemetry.Extensions.Hosting.Tests" + AssemblyInfo.PublicKey)] [assembly: InternalsVisibleTo("DynamicProxyGenAssembly2" + AssemblyInfo.MoqPublicKey)] [assembly: InternalsVisibleTo("Benchmarks" + AssemblyInfo.PublicKey)] - -// TODO: Much of the metrics SDK is currently internal. These should be removed once the public API surface area for metrics is defined. -[assembly: InternalsVisibleTo("OpenTelemetry.Exporter.OpenTelemetryProtocol" + AssemblyInfo.PublicKey)] -[assembly: InternalsVisibleTo("OpenTelemetry.Exporter.OpenTelemetryProtocol.Tests" + AssemblyInfo.PublicKey)] diff --git a/test/OpenTelemetry.Exporter.OpenTelemetryProtocol.Tests/OtlpMetricsExporterTests.cs b/test/OpenTelemetry.Exporter.OpenTelemetryProtocol.Tests/OtlpMetricsExporterTests.cs index 981b7c1919a..541a714b62f 100644 --- a/test/OpenTelemetry.Exporter.OpenTelemetryProtocol.Tests/OtlpMetricsExporterTests.cs +++ b/test/OpenTelemetry.Exporter.OpenTelemetryProtocol.Tests/OtlpMetricsExporterTests.cs @@ -38,10 +38,6 @@ public class OtlpMetricsExporterTests [InlineData(false)] public void ToOtlpResourceMetricsTest(bool includeServiceNameInResource) { - using var exporter = new OtlpMetricsExporter( - new OtlpExporterOptions(), - new NoopMetricsServiceClient()); - var resourceBuilder = ResourceBuilder.CreateEmpty(); if (includeServiceNameInResource) { @@ -70,8 +66,6 @@ public void ToOtlpResourceMetricsTest(bool includeServiceNameInResource) .AddReader(metricReader) .Build(); - exporter.ParentProvider = provider; - using var meter = new Meter("TestMeter", "0.0.1"); var counter = meter.CreateCounter("counter"); @@ -88,7 +82,7 @@ public void ToOtlpResourceMetricsTest(bool includeServiceNameInResource) void RunTest(Batch metrics) { var request = new OtlpCollector.ExportMetricsServiceRequest(); - request.AddMetrics(exporter.ProcessResource, metrics); + request.AddMetrics(resourceBuilder.Build().ToOtlpResource(), metrics); Assert.Single(request.ResourceMetrics); var resourceMetric = request.ResourceMetrics.First(); diff --git a/test/OpenTelemetry.Exporter.OpenTelemetryProtocol.Tests/OtlpTraceExporterTests.cs b/test/OpenTelemetry.Exporter.OpenTelemetryProtocol.Tests/OtlpTraceExporterTests.cs index c1ae07ca70a..1964bd326a2 100644 --- a/test/OpenTelemetry.Exporter.OpenTelemetryProtocol.Tests/OtlpTraceExporterTests.cs +++ b/test/OpenTelemetry.Exporter.OpenTelemetryProtocol.Tests/OtlpTraceExporterTests.cs @@ -68,10 +68,6 @@ public void ToOtlpResourceSpansTest(bool includeServiceNameInResource) new ActivitySource("odd", "1.3.5"), }; - using var exporter = new OtlpTraceExporter( - new OtlpExporterOptions(), - new NoopTraceServiceClient()); - var resourceBuilder = ResourceBuilder.CreateEmpty(); if (includeServiceNameInResource) { @@ -90,8 +86,6 @@ public void ToOtlpResourceSpansTest(bool includeServiceNameInResource) using var openTelemetrySdk = builder.Build(); - exporter.ParentProvider = openTelemetrySdk; - var processor = new BatchActivityExportProcessor(new TestExporter(RunTest)); const int numOfSpans = 10; bool isEven; @@ -112,7 +106,7 @@ void RunTest(Batch batch) { var request = new OtlpCollector.ExportTraceServiceRequest(); - request.AddBatch(exporter.ProcessResource, batch); + request.AddBatch(resourceBuilder.Build().ToOtlpResource(), batch); Assert.Single(request.ResourceSpans); var oltpResource = request.ResourceSpans.First().Resource; From 2e8dd4d49cf3f4fd4f64cd622a963268b9811245 Mon Sep 17 00:00:00 2001 From: Cijo Thomas Date: Thu, 30 Sep 2021 16:51:51 -0700 Subject: [PATCH 05/39] Extract common logic to own methods in AggregatorStore (#2435) --- src/OpenTelemetry/Metrics/AggregatorStore.cs | 187 +++++++++--------- src/OpenTelemetry/Metrics/MeterProviderSdk.cs | 7 +- .../Metrics/ThreadStaticStorage.cs | 21 +- 3 files changed, 111 insertions(+), 104 deletions(-) diff --git a/src/OpenTelemetry/Metrics/AggregatorStore.cs b/src/OpenTelemetry/Metrics/AggregatorStore.cs index 0d99cf61442..8bccd5ef3e7 100644 --- a/src/OpenTelemetry/Metrics/AggregatorStore.cs +++ b/src/OpenTelemetry/Metrics/AggregatorStore.cs @@ -17,6 +17,7 @@ using System; using System.Collections.Concurrent; using System.Collections.Generic; +using System.Runtime.CompilerServices; using System.Threading; namespace OpenTelemetry.Metrics @@ -51,105 +52,23 @@ internal AggregatorStore(AggregationType aggType, AggregationTemporality tempora internal int FindMetricAggregators(ReadOnlySpan> tags) { - int len = tags.Length; - if (len == 0) + int tagLength = tags.Length; + if (tagLength == 0) { - if (!this.zeroTagMetricPointInitialized) - { - lock (this.lockZeroTags) - { - if (!this.zeroTagMetricPointInitialized) - { - var dt = DateTimeOffset.UtcNow; - this.metrics[0] = new MetricPoint(this.aggType, dt, null, null, this.histogramBounds); - this.zeroTagMetricPointInitialized = true; - } - } - } - + this.InitializeZeroTagPointIfNotInitialized(); return 0; } var storage = ThreadStaticStorage.GetStorage(); - storage.SplitToKeysAndValues(tags, out var tagKey, out var tagValue); + storage.SplitToKeysAndValues(tags, tagLength, out var tagKey, out var tagValue); - if (len > 1) + if (tagLength > 1) { Array.Sort(tagKey, tagValue); } - int aggregatorIndex; - - string[] seqKey = null; - - // GetOrAdd by TagKey at 1st Level of 2-level dictionary structure. - // Get back a Dictionary of [ Values x Metrics[] ]. - if (!this.keyValue2MetricAggs.TryGetValue(tagKey, out var value2metrics)) - { - // Note: We are using storage from ThreadStatic, so need to make a deep copy for Dictionary storage. - seqKey = new string[len]; - tagKey.CopyTo(seqKey, 0); - - value2metrics = new ConcurrentDictionary(ObjectArrayComparer); - if (!this.keyValue2MetricAggs.TryAdd(seqKey, value2metrics)) - { - this.keyValue2MetricAggs.TryGetValue(seqKey, out value2metrics); - } - } - - // GetOrAdd by TagValue at 2st Level of 2-level dictionary structure. - // Get back Metrics[]. - if (!value2metrics.TryGetValue(tagValue, out aggregatorIndex)) - { - aggregatorIndex = this.metricPointIndex; - if (aggregatorIndex >= MaxMetricPoints) - { - // sorry! out of data points. - // TODO: Once we support cleanup of - // unused points (typically with delta) - // we can re-claim them here. - return -1; - } - - lock (value2metrics) - { - // check again after acquiring lock. - if (!value2metrics.TryGetValue(tagValue, out aggregatorIndex)) - { - aggregatorIndex = Interlocked.Increment(ref this.metricPointIndex); - if (aggregatorIndex >= MaxMetricPoints) - { - // sorry! out of data points. - // TODO: Once we support cleanup of - // unused points (typically with delta) - // we can re-claim them here. - return -1; - } - - // Note: We are using storage from ThreadStatic, so need to make a deep copy for Dictionary storage. - if (seqKey == null) - { - seqKey = new string[len]; - tagKey.CopyTo(seqKey, 0); - } - - var seqVal = new object[len]; - tagValue.CopyTo(seqVal, 0); - - ref var metricPoint = ref this.metrics[aggregatorIndex]; - var dt = DateTimeOffset.UtcNow; - metricPoint = new MetricPoint(this.aggType, dt, seqKey, seqVal, this.histogramBounds); - - // Add to dictionary *after* initializing MetricPoint - // as other threads can start writing to the - // MetricPoint, if dictionary entry found. - value2metrics.TryAdd(seqVal, aggregatorIndex); - } - } - } - - return aggregatorIndex; + return this.LookupAggregatorStore(tagKey, tagValue, tagLength); } internal void UpdateLong(long value, ReadOnlySpan> tags) @@ -222,5 +141,97 @@ internal BatchMetricPoint GetMetricPoints() var indexSnapShot = Math.Min(this.metricPointIndex, MaxMetricPoints - 1); return new BatchMetricPoint(this.metrics, indexSnapShot + 1, this.startTimeExclusive, this.endTimeInclusive); } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void InitializeZeroTagPointIfNotInitialized() + { + if (!this.zeroTagMetricPointInitialized) + { + lock (this.lockZeroTags) + { + if (!this.zeroTagMetricPointInitialized) + { + var dt = DateTimeOffset.UtcNow; + this.metrics[0] = new MetricPoint(this.aggType, dt, null, null, this.histogramBounds); + this.zeroTagMetricPointInitialized = true; + } + } + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private int LookupAggregatorStore(string[] tagKey, object[] tagValue, int length) + { + int aggregatorIndex; + string[] seqKey = null; + + // GetOrAdd by TagKey at 1st Level of 2-level dictionary structure. + // Get back a Dictionary of [ Values x Metrics[] ]. + if (!this.keyValue2MetricAggs.TryGetValue(tagKey, out var value2metrics)) + { + // Note: We are using storage from ThreadStatic, so need to make a deep copy for Dictionary storage. + seqKey = new string[length]; + tagKey.CopyTo(seqKey, 0); + + value2metrics = new ConcurrentDictionary(ObjectArrayComparer); + if (!this.keyValue2MetricAggs.TryAdd(seqKey, value2metrics)) + { + this.keyValue2MetricAggs.TryGetValue(seqKey, out value2metrics); + } + } + + // GetOrAdd by TagValue at 2st Level of 2-level dictionary structure. + // Get back Metrics[]. + if (!value2metrics.TryGetValue(tagValue, out aggregatorIndex)) + { + aggregatorIndex = this.metricPointIndex; + if (aggregatorIndex >= MaxMetricPoints) + { + // sorry! out of data points. + // TODO: Once we support cleanup of + // unused points (typically with delta) + // we can re-claim them here. + return -1; + } + + lock (value2metrics) + { + // check again after acquiring lock. + if (!value2metrics.TryGetValue(tagValue, out aggregatorIndex)) + { + aggregatorIndex = Interlocked.Increment(ref this.metricPointIndex); + if (aggregatorIndex >= MaxMetricPoints) + { + // sorry! out of data points. + // TODO: Once we support cleanup of + // unused points (typically with delta) + // we can re-claim them here. + return -1; + } + + // Note: We are using storage from ThreadStatic, so need to make a deep copy for Dictionary storage. + if (seqKey == null) + { + seqKey = new string[length]; + tagKey.CopyTo(seqKey, 0); + } + + var seqVal = new object[length]; + tagValue.CopyTo(seqVal, 0); + + ref var metricPoint = ref this.metrics[aggregatorIndex]; + var dt = DateTimeOffset.UtcNow; + metricPoint = new MetricPoint(this.aggType, dt, seqKey, seqVal, this.histogramBounds); + + // Add to dictionary *after* initializing MetricPoint + // as other threads can start writing to the + // MetricPoint, if dictionary entry found. + value2metrics.TryAdd(seqVal, aggregatorIndex); + } + } + } + + return aggregatorIndex; + } } } diff --git a/src/OpenTelemetry/Metrics/MeterProviderSdk.cs b/src/OpenTelemetry/Metrics/MeterProviderSdk.cs index 81be9ae72d9..1494bd27635 100644 --- a/src/OpenTelemetry/Metrics/MeterProviderSdk.cs +++ b/src/OpenTelemetry/Metrics/MeterProviderSdk.cs @@ -16,6 +16,7 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.Diagnostics.Metrics; using System.Linq; using OpenTelemetry.Resources; @@ -254,7 +255,8 @@ internal void MeasurementRecordedDouble(Instrument instrument, double value, Rea // of size one as state. var metrics = state as List; - if (instrument == null || metrics == null) + Debug.Assert(instrument != null, "instrument must be non-null."); + if (metrics == null) { // TODO: log return; @@ -271,7 +273,8 @@ internal void MeasurementRecordedLong(Instrument instrument, long value, ReadOnl // Get Instrument State var metrics = state as List; - if (instrument == null || metrics == null) + Debug.Assert(instrument != null, "instrument must be non-null."); + if (metrics == null) { // TODO: log return; diff --git a/src/OpenTelemetry/Metrics/ThreadStaticStorage.cs b/src/OpenTelemetry/Metrics/ThreadStaticStorage.cs index ba3d2f3bd9f..3e2dd0be834 100644 --- a/src/OpenTelemetry/Metrics/ThreadStaticStorage.cs +++ b/src/OpenTelemetry/Metrics/ThreadStaticStorage.cs @@ -49,27 +49,20 @@ internal static ThreadStaticStorage GetStorage() } [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal void SplitToKeysAndValues(ReadOnlySpan> tags, out string[] tagKeys, out object[] tagValues) + internal void SplitToKeysAndValues(ReadOnlySpan> tags, int tagLength, out string[] tagKeys, out object[] tagValues) { - var len = tags.Length; - - if (len == 0) - { - throw new InvalidOperationException("There must be atleast one tag to use ThreadStaticStorage."); - } - - if (len <= MaxTagCacheSize) + if (tagLength <= MaxTagCacheSize) { - tagKeys = this.tagStorage[len - 1].TagKey; - tagValues = this.tagStorage[len - 1].TagValue; + tagKeys = this.tagStorage[tagLength - 1].TagKey; + tagValues = this.tagStorage[tagLength - 1].TagValue; } else { - tagKeys = new string[len]; - tagValues = new object[len]; + tagKeys = new string[tagLength]; + tagValues = new object[tagLength]; } - for (var n = 0; n < len; n++) + for (var n = 0; n < tagLength; n++) { tagKeys[n] = tags[n].Key; tagValues[n] = tags[n].Value; From 1e4094e131474ebb74c1fb507a4677145056cf64 Mon Sep 17 00:00:00 2001 From: Cijo Thomas Date: Fri, 1 Oct 2021 10:00:12 -0700 Subject: [PATCH 06/39] Allow description override using View (#2438) --- .../ConsoleMetricExporter.cs | 2 +- src/OpenTelemetry/Metrics/MeterProviderSdk.cs | 7 +++--- src/OpenTelemetry/Metrics/Metric.cs | 22 ++++++++++--------- .../Metrics/MetricStreamConfiguration.cs | 2 -- .../Metrics/MetricViewTests.cs | 8 ++++--- 5 files changed, 22 insertions(+), 19 deletions(-) diff --git a/src/OpenTelemetry.Exporter.Console/ConsoleMetricExporter.cs b/src/OpenTelemetry.Exporter.Console/ConsoleMetricExporter.cs index 49092a3187d..06e416fdcb5 100644 --- a/src/OpenTelemetry.Exporter.Console/ConsoleMetricExporter.cs +++ b/src/OpenTelemetry.Exporter.Console/ConsoleMetricExporter.cs @@ -55,7 +55,7 @@ public override ExportResult Export(in Batch batch) msg.Append(metric.Name); if (!string.IsNullOrEmpty(metric.Description)) { - msg.Append(' '); + msg.Append(", "); msg.Append(metric.Description); } diff --git a/src/OpenTelemetry/Metrics/MeterProviderSdk.cs b/src/OpenTelemetry/Metrics/MeterProviderSdk.cs index 1494bd27635..004dd10f135 100644 --- a/src/OpenTelemetry/Metrics/MeterProviderSdk.cs +++ b/src/OpenTelemetry/Metrics/MeterProviderSdk.cs @@ -154,14 +154,15 @@ internal MeterProviderSdk( else { Metric metric; + var metricDescription = metricStreamConfig?.Description ?? instrument.Description; if (metricStreamConfig is HistogramConfiguration histogramConfig && histogramConfig.BucketBounds != null) { - metric = new Metric(instrument, temporality, histogramConfig.BucketBounds, metricStreamName); + metric = new Metric(instrument, temporality, metricStreamName, metricDescription, histogramConfig.BucketBounds); } else { - metric = new Metric(instrument, temporality, metricStreamName); + metric = new Metric(instrument, temporality, metricStreamName, metricDescription); } this.metrics[index] = metric; @@ -206,7 +207,7 @@ internal MeterProviderSdk( else { metrics = new List(1); - var metric = new Metric(instrument, temporality); + var metric = new Metric(instrument, temporality, metricName, instrument.Description); this.metrics[index] = metric; metrics.Add(metric); this.metricStreamNames.Add(metricName, true); diff --git a/src/OpenTelemetry/Metrics/Metric.cs b/src/OpenTelemetry/Metrics/Metric.cs index ca3ebff5375..a05402e4e9c 100644 --- a/src/OpenTelemetry/Metrics/Metric.cs +++ b/src/OpenTelemetry/Metrics/Metric.cs @@ -25,15 +25,15 @@ public class Metric internal static readonly double[] DefaultHistogramBounds = new double[] { 0, 5, 10, 25, 50, 75, 100, 250, 500, 1000 }; private AggregatorStore aggStore; - internal Metric(Instrument instrument, AggregationTemporality temporality, string metricName = "") - : this(instrument, temporality, DefaultHistogramBounds, metricName) + internal Metric( + Instrument instrument, + AggregationTemporality temporality, + string metricName, + string metricDescription, + double[] histogramBounds = null) { - } - - internal Metric(Instrument instrument, AggregationTemporality temporality, double[] histogramBounds, string metricName = "") - { - this.Name = string.IsNullOrWhiteSpace(metricName) ? instrument.Name : metricName; - this.Description = instrument.Description; + this.Name = metricName; + this.Description = metricDescription; this.Unit = instrument.Unit; this.Meter = instrument.Meter; AggregationType aggType = default; @@ -87,7 +87,9 @@ internal Metric(Instrument instrument, AggregationTemporality temporality, doubl || instrument.GetType() == typeof(Histogram)) { this.MetricType = MetricType.Histogram; - if (histogramBounds.Length == 0) + + if (histogramBounds != null + && histogramBounds.Length == 0) { aggType = AggregationType.HistogramSumCount; } @@ -101,7 +103,7 @@ internal Metric(Instrument instrument, AggregationTemporality temporality, doubl // TODO: Log and assign some invalid Enum. } - this.aggStore = new AggregatorStore(aggType, temporality, histogramBounds); + this.aggStore = new AggregatorStore(aggType, temporality, histogramBounds ?? DefaultHistogramBounds); this.Temporality = temporality; } diff --git a/src/OpenTelemetry/Metrics/MetricStreamConfiguration.cs b/src/OpenTelemetry/Metrics/MetricStreamConfiguration.cs index 154cab306a8..52aa3cac003 100644 --- a/src/OpenTelemetry/Metrics/MetricStreamConfiguration.cs +++ b/src/OpenTelemetry/Metrics/MetricStreamConfiguration.cs @@ -35,8 +35,6 @@ public class MetricStreamConfiguration public string Description { get; set; } - public string Unit { get; set; } - public string[] TagKeys { get; set; } public virtual Aggregation Aggregation { get; set; } diff --git a/test/OpenTelemetry.Tests/Metrics/MetricViewTests.cs b/test/OpenTelemetry.Tests/Metrics/MetricViewTests.cs index 2e6313d871a..bb5510e213d 100644 --- a/test/OpenTelemetry.Tests/Metrics/MetricViewTests.cs +++ b/test/OpenTelemetry.Tests/Metrics/MetricViewTests.cs @@ -66,7 +66,7 @@ public void ViewToRenameMetricConditionally() if (instrument.Meter.Name.Equals(meter2.Name, StringComparison.OrdinalIgnoreCase) && instrument.Name.Equals("name1", StringComparison.OrdinalIgnoreCase)) { - return new MetricStreamConfiguration() { Name = "name1_Renamed" }; + return new MetricStreamConfiguration() { Name = "name1_Renamed", Description = "new description" }; } else { @@ -80,14 +80,16 @@ public void ViewToRenameMetricConditionally() // exported (the 2nd one gets dropped due to // name conflict). Due to renaming with Views, // we expect 2 metric streams here. - var counter1 = meter1.CreateCounter("name1"); - var counter2 = meter2.CreateCounter("name1"); + var counter1 = meter1.CreateCounter("name1", "unit", "original_description"); + var counter2 = meter2.CreateCounter("name1", "unit", "original_description"); counter1.Add(10); counter2.Add(10); meterProvider.ForceFlush(MaxTimeToAllowForFlush); Assert.Equal(2, exportedItems.Count); Assert.Equal("name1", exportedItems[0].Name); Assert.Equal("name1_Renamed", exportedItems[1].Name); + Assert.Equal("original_description", exportedItems[0].Description); + Assert.Equal("new description", exportedItems[1].Description); } [Fact] From 66329878f3fd202c2afc3a23b5a2a04205a72cd3 Mon Sep 17 00:00:00 2001 From: Tom Tan Date: Fri, 1 Oct 2021 10:11:13 -0700 Subject: [PATCH 07/39] Replace '\n' with a call to Console.Write() (#2437) --- test/OpenTelemetry.Tests.Stress/Skeleton.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/OpenTelemetry.Tests.Stress/Skeleton.cs b/test/OpenTelemetry.Tests.Stress/Skeleton.cs index 5033abf903a..d8a3bf4dcc5 100644 --- a/test/OpenTelemetry.Tests.Stress/Skeleton.cs +++ b/test/OpenTelemetry.Tests.Stress/Skeleton.cs @@ -31,7 +31,8 @@ public partial class Program public static void Stress(int concurrency = 0) { #if DEBUG - Console.WriteLine("***WARNING*** The current build is DEBUG which may affect timing!\n"); + Console.WriteLine("***WARNING*** The current build is DEBUG which may affect timing!"); + Console.WriteLine(); #endif if (concurrency < 0) From 2c2515b7743362b5a8e28a9120792701ba015ddf Mon Sep 17 00:00:00 2001 From: Mikel Blanchard Date: Fri, 1 Oct 2021 11:30:22 -0700 Subject: [PATCH 08/39] Prometheus: Create unit test project (#2440) --- OpenTelemetry.sln | 6 ++ .../AssemblyInfo.cs | 2 + .../PrometheusExporterExtensions.cs | 8 +- .../Implementation/PrometheusMetricBuilder.cs | 8 +- ...Telemetry.Exporter.Prometheus.Tests.csproj | 29 +++++++ .../PrometheusExporterExtensionsTests.cs | 77 +++++++++++++++++++ 6 files changed, 127 insertions(+), 3 deletions(-) create mode 100644 test/OpenTelemetry.Exporter.Prometheus.Tests/OpenTelemetry.Exporter.Prometheus.Tests.csproj create mode 100644 test/OpenTelemetry.Exporter.Prometheus.Tests/PrometheusExporterExtensionsTests.cs diff --git a/OpenTelemetry.sln b/OpenTelemetry.sln index 3b705899e85..22e4bddf798 100644 --- a/OpenTelemetry.sln +++ b/OpenTelemetry.sln @@ -218,6 +218,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "customizing-the-sdk", "docs EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OpenTelemetry.Tests.Stress", "test\OpenTelemetry.Tests.Stress\OpenTelemetry.Tests.Stress.csproj", "{2770158A-D220-414B-ABC6-179371323579}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "OpenTelemetry.Exporter.Prometheus.Tests", "test\OpenTelemetry.Exporter.Prometheus.Tests\OpenTelemetry.Exporter.Prometheus.Tests.csproj", "{380EE686-91F1-45B3-AEEB-755F0E5B068F}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -448,6 +450,10 @@ Global {2770158A-D220-414B-ABC6-179371323579}.Debug|Any CPU.Build.0 = Debug|Any CPU {2770158A-D220-414B-ABC6-179371323579}.Release|Any CPU.ActiveCfg = Release|Any CPU {2770158A-D220-414B-ABC6-179371323579}.Release|Any CPU.Build.0 = Release|Any CPU + {380EE686-91F1-45B3-AEEB-755F0E5B068F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {380EE686-91F1-45B3-AEEB-755F0E5B068F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {380EE686-91F1-45B3-AEEB-755F0E5B068F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {380EE686-91F1-45B3-AEEB-755F0E5B068F}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/src/OpenTelemetry.Exporter.Prometheus/AssemblyInfo.cs b/src/OpenTelemetry.Exporter.Prometheus/AssemblyInfo.cs index 36f813dca15..fd06ffcdfaa 100644 --- a/src/OpenTelemetry.Exporter.Prometheus/AssemblyInfo.cs +++ b/src/OpenTelemetry.Exporter.Prometheus/AssemblyInfo.cs @@ -17,6 +17,8 @@ #if SIGNED [assembly: InternalsVisibleTo("Benchmarks, PublicKey=002400000480000094000000060200000024000052534131000400000100010051c1562a090fb0c9f391012a32198b5e5d9a60e9b80fa2d7b434c9e5ccb7259bd606e66f9660676afc6692b8cdc6793d190904551d2103b7b22fa636dcbb8208839785ba402ea08fc00c8f1500ccef28bbf599aa64ffb1e1d5dc1bf3420a3777badfe697856e9d52070a50c3ea5821c80bef17ca3acffa28f89dd413f096f898")] +[assembly: InternalsVisibleTo("OpenTelemetry.Exporter.Prometheus.Tests, PublicKey=002400000480000094000000060200000024000052534131000400000100010051c1562a090fb0c9f391012a32198b5e5d9a60e9b80fa2d7b434c9e5ccb7259bd606e66f9660676afc6692b8cdc6793d190904551d2103b7b22fa636dcbb8208839785ba402ea08fc00c8f1500ccef28bbf599aa64ffb1e1d5dc1bf3420a3777badfe697856e9d52070a50c3ea5821c80bef17ca3acffa28f89dd413f096f898")] #else [assembly: InternalsVisibleTo("Benchmarks")] +[assembly: InternalsVisibleTo("OpenTelemetry.Exporter.Prometheus.Tests")] #endif diff --git a/src/OpenTelemetry.Exporter.Prometheus/Implementation/PrometheusExporterExtensions.cs b/src/OpenTelemetry.Exporter.Prometheus/Implementation/PrometheusExporterExtensions.cs index 74d5c4345a9..1b14627551c 100644 --- a/src/OpenTelemetry.Exporter.Prometheus/Implementation/PrometheusExporterExtensions.cs +++ b/src/OpenTelemetry.Exporter.Prometheus/Implementation/PrometheusExporterExtensions.cs @@ -14,6 +14,7 @@ // limitations under the License. // +using System; using System.Globalization; using System.IO; using System.Threading.Tasks; @@ -36,17 +37,20 @@ internal static class PrometheusExporterExtensions private const string PrometheusHistogramBucketLabelPositiveInfinity = "+Inf"; private const string PrometheusHistogramBucketLabelLessThan = "le"; + private static readonly Func DefaultGetUtcNowDateTimeOffset = () => DateTimeOffset.UtcNow; + /// /// Serialize metrics to prometheus format. /// /// . /// StreamWriter to write to. + /// Optional function to resolve the current date & time. /// to await the operation. - public static async Task WriteMetricsCollection(this PrometheusExporter exporter, StreamWriter writer) + public static async Task WriteMetricsCollection(this PrometheusExporter exporter, StreamWriter writer, Func getUtcNowDateTimeOffset = null) { foreach (var metric in exporter.Metrics) { - var builder = new PrometheusMetricBuilder() + var builder = new PrometheusMetricBuilder(getUtcNowDateTimeOffset ?? DefaultGetUtcNowDateTimeOffset) .WithName(metric.Name) .WithDescription(metric.Description); diff --git a/src/OpenTelemetry.Exporter.Prometheus/Implementation/PrometheusMetricBuilder.cs b/src/OpenTelemetry.Exporter.Prometheus/Implementation/PrometheusMetricBuilder.cs index 751a7be50ed..d4166a1f57b 100644 --- a/src/OpenTelemetry.Exporter.Prometheus/Implementation/PrometheusMetricBuilder.cs +++ b/src/OpenTelemetry.Exporter.Prometheus/Implementation/PrometheusMetricBuilder.cs @@ -59,11 +59,17 @@ internal sealed class PrometheusMetricBuilder }; private readonly ICollection values = new List(); + private readonly Func getUtcNowDateTimeOffset; private string name; private string description; private string type; + public PrometheusMetricBuilder(Func getUtcNowDateTimeOffset) + { + this.getUtcNowDateTimeOffset = getUtcNowDateTimeOffset; + } + public PrometheusMetricBuilder WithName(string name) { this.name = name; @@ -141,7 +147,7 @@ public async Task Write(StreamWriter writer) // ] value [ timestamp ] // In the sample syntax: - var now = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds().ToString(CultureInfo.InvariantCulture); + var now = this.getUtcNowDateTimeOffset().ToUnixTimeMilliseconds().ToString(CultureInfo.InvariantCulture); foreach (var m in this.values) { diff --git a/test/OpenTelemetry.Exporter.Prometheus.Tests/OpenTelemetry.Exporter.Prometheus.Tests.csproj b/test/OpenTelemetry.Exporter.Prometheus.Tests/OpenTelemetry.Exporter.Prometheus.Tests.csproj new file mode 100644 index 00000000000..bed1d487aee --- /dev/null +++ b/test/OpenTelemetry.Exporter.Prometheus.Tests/OpenTelemetry.Exporter.Prometheus.Tests.csproj @@ -0,0 +1,29 @@ + + + Unit test project for Prometheus Exporter for OpenTelemetry + netcoreapp3.1;net5.0 + $(TargetFrameworks);net461 + + false + + + + + + + + all + runtime; build; native; contentfiles; analyzers + + + + + + + + + + + + + diff --git a/test/OpenTelemetry.Exporter.Prometheus.Tests/PrometheusExporterExtensionsTests.cs b/test/OpenTelemetry.Exporter.Prometheus.Tests/PrometheusExporterExtensionsTests.cs new file mode 100644 index 00000000000..3762ddb13aa --- /dev/null +++ b/test/OpenTelemetry.Exporter.Prometheus.Tests/PrometheusExporterExtensionsTests.cs @@ -0,0 +1,77 @@ +// +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +using System; +using System.Collections.Generic; +using System.Diagnostics.Metrics; +using System.IO; +using System.Text; +using OpenTelemetry.Metrics; +using OpenTelemetry.Tests; +using Xunit; + +namespace OpenTelemetry.Exporter.Prometheus.Tests +{ + public sealed class PrometheusExporterExtensionsTests + { + [Fact] + public void WriteMetricsCollectionTest() + { + var tags = new KeyValuePair[] + { + new KeyValuePair("key1", "value1"), + new KeyValuePair("key2", "value2"), + }; + + using var provider = Sdk.CreateMeterProviderBuilder() + .AddSource("TestMeter") + .AddReader(new BaseExportingMetricReader(new TestExporter(RunTest))) + .Build(); + + using var meter = new Meter("TestMeter", "0.0.1"); + + var counter = meter.CreateCounter("counter"); + + counter.Add(100, tags); + + var testCompleted = false; + + // Invokes the TestExporter which will invoke RunTest + provider.ForceFlush(3000); + + Assert.True(testCompleted); + + void RunTest(Batch metrics) + { + using PrometheusExporter prometheusExporter = new PrometheusExporter(new PrometheusExporterOptions()); + + prometheusExporter.Metrics = metrics; + + using MemoryStream ms = new MemoryStream(); + using (StreamWriter writer = new StreamWriter(ms)) + { + PrometheusExporterExtensions.WriteMetricsCollection(prometheusExporter, writer, () => new DateTimeOffset(2021, 9, 30, 22, 30, 0, TimeSpan.Zero)).GetAwaiter().GetResult(); + } + + Assert.Equal( + "# TYPE counter counter\ncounter{key1=\"value1\",key2=\"value2\"} 100 1633041000000\n", + Encoding.UTF8.GetString(ms.ToArray())); + + testCompleted = true; + } + } + } +} From 793d18ef025e3cf08613ee5d5c80556c11bc980c Mon Sep 17 00:00:00 2001 From: Cijo Thomas Date: Fri, 1 Oct 2021 12:26:22 -0700 Subject: [PATCH 09/39] Use instrument state as single metric instead of list with size one (#2441) --- src/OpenTelemetry/Metrics/MeterProviderSdk.cs | 103 +++++++++++++----- 1 file changed, 77 insertions(+), 26 deletions(-) diff --git a/src/OpenTelemetry/Metrics/MeterProviderSdk.cs b/src/OpenTelemetry/Metrics/MeterProviderSdk.cs index 004dd10f135..3498aa4b308 100644 --- a/src/OpenTelemetry/Metrics/MeterProviderSdk.cs +++ b/src/OpenTelemetry/Metrics/MeterProviderSdk.cs @@ -178,6 +178,16 @@ internal MeterProviderSdk( } } }; + + // Everything double + this.listener.SetMeasurementEventCallback(this.MeasurementRecordedDouble); + this.listener.SetMeasurementEventCallback((instrument, value, tags, state) => this.MeasurementRecordedDouble(instrument, value, tags, state)); + + // Everything long + this.listener.SetMeasurementEventCallback(this.MeasurementRecordedLong); + this.listener.SetMeasurementEventCallback((instrument, value, tags, state) => this.MeasurementRecordedLong(instrument, value, tags, state)); + this.listener.SetMeasurementEventCallback((instrument, value, tags, state) => this.MeasurementRecordedLong(instrument, value, tags, state)); + this.listener.SetMeasurementEventCallback((instrument, value, tags, state) => this.MeasurementRecordedLong(instrument, value, tags, state)); } else { @@ -186,7 +196,7 @@ internal MeterProviderSdk( if (meterSourcesToSubscribe.ContainsKey(instrument.Meter.Name)) { var metricName = instrument.Name; - List metrics = null; + Metric metric = null; lock (this.instrumentCreationLock) { if (this.metricStreamNames.ContainsKey(metricName)) @@ -206,31 +216,28 @@ internal MeterProviderSdk( } else { - metrics = new List(1); - var metric = new Metric(instrument, temporality, metricName, instrument.Description); + metric = new Metric(instrument, temporality, metricName, instrument.Description); this.metrics[index] = metric; - metrics.Add(metric); this.metricStreamNames.Add(metricName, true); } } - listener.EnableMeasurementEvents(instrument, metrics); + listener.EnableMeasurementEvents(instrument, metric); } }; - } - - this.listener.MeasurementsCompleted = (instrument, state) => this.MeasurementsCompleted(instrument, state); - // Everything double - this.listener.SetMeasurementEventCallback((instrument, value, tags, state) => this.MeasurementRecordedDouble(instrument, value, tags, state)); - this.listener.SetMeasurementEventCallback((instrument, value, tags, state) => this.MeasurementRecordedDouble(instrument, value, tags, state)); + // Everything double + this.listener.SetMeasurementEventCallback(this.MeasurementRecordedDoubleSingleStream); + this.listener.SetMeasurementEventCallback((instrument, value, tags, state) => this.MeasurementRecordedDoubleSingleStream(instrument, value, tags, state)); - // Everything long - this.listener.SetMeasurementEventCallback((instrument, value, tags, state) => this.MeasurementRecordedLong(instrument, value, tags, state)); - this.listener.SetMeasurementEventCallback((instrument, value, tags, state) => this.MeasurementRecordedLong(instrument, value, tags, state)); - this.listener.SetMeasurementEventCallback((instrument, value, tags, state) => this.MeasurementRecordedLong(instrument, value, tags, state)); - this.listener.SetMeasurementEventCallback((instrument, value, tags, state) => this.MeasurementRecordedLong(instrument, value, tags, state)); + // Everything long + this.listener.SetMeasurementEventCallback(this.MeasurementRecordedLongSingleStream); + this.listener.SetMeasurementEventCallback((instrument, value, tags, state) => this.MeasurementRecordedLongSingleStream(instrument, value, tags, state)); + this.listener.SetMeasurementEventCallback((instrument, value, tags, state) => this.MeasurementRecordedLongSingleStream(instrument, value, tags, state)); + this.listener.SetMeasurementEventCallback((instrument, value, tags, state) => this.MeasurementRecordedLongSingleStream(instrument, value, tags, state)); + } + this.listener.MeasurementsCompleted = (instrument, state) => this.MeasurementsCompleted(instrument, state); this.listener.Start(); } @@ -248,12 +255,6 @@ internal void MeasurementsCompleted(Instrument instrument, object state) internal void MeasurementRecordedDouble(Instrument instrument, double value, ReadOnlySpan> tagsRos, object state) { // Get Instrument State - // TODO: Benchmark and see if it makes - // sense to use a different state - // when there are no views registered. - // In that case, storing Metric as state - // might be faster than storing List - // of size one as state. var metrics = state as List; Debug.Assert(instrument != null, "instrument must be non-null."); @@ -263,9 +264,19 @@ internal void MeasurementRecordedDouble(Instrument instrument, double value, Rea return; } - foreach (var metric in metrics) + if (metrics.Count == 1) + { + // special casing the common path + // as this is faster than the + // foreach, when count is 1. + metrics[0].UpdateDouble(value, tagsRos); + } + else { - metric.UpdateDouble(value, tagsRos); + foreach (var metric in metrics) + { + metric.UpdateDouble(value, tagsRos); + } } } @@ -281,12 +292,52 @@ internal void MeasurementRecordedLong(Instrument instrument, long value, ReadOnl return; } - foreach (var metric in metrics) + if (metrics.Count == 1) + { + // special casing the common path + // as this is faster than the + // foreach, when count is 1. + metrics[0].UpdateLong(value, tagsRos); + } + else { - metric.UpdateLong(value, tagsRos); + foreach (var metric in metrics) + { + metric.UpdateLong(value, tagsRos); + } } } + internal void MeasurementRecordedLongSingleStream(Instrument instrument, long value, ReadOnlySpan> tagsRos, object state) + { + // Get Instrument State + var metric = state as Metric; + + Debug.Assert(instrument != null, "instrument must be non-null."); + if (metric == null) + { + // TODO: log + return; + } + + metric.UpdateLong(value, tagsRos); + } + + internal void MeasurementRecordedDoubleSingleStream(Instrument instrument, double value, ReadOnlySpan> tagsRos, object state) + { + // Get Instrument State + var metric = state as Metric; + + Debug.Assert(instrument != null, "instrument must be non-null."); + if (metric == null) + { + // TODO: log + return; + } + + metric.UpdateDouble(value, tagsRos); + } + internal Batch Collect() { lock (this.collectLock) From bd189aa654948e79203d66d4b6d33fb93557d7e9 Mon Sep 17 00:00:00 2001 From: Alan West <3676547+alanwest@users.noreply.github.com> Date: Fri, 1 Oct 2021 13:55:35 -0700 Subject: [PATCH 10/39] Rename OtlpMetricsExporter -> OtlpMetricExporter (#2443) --- ...{OtlpMetricsExporter.cs => OtlpMetricExporter.cs} | 12 ++++++------ .../OtlpMetricExporterExtensions.cs | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) rename src/OpenTelemetry.Exporter.OpenTelemetryProtocol/{OtlpMetricsExporter.cs => OtlpMetricExporter.cs} (90%) diff --git a/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/OtlpMetricsExporter.cs b/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/OtlpMetricExporter.cs similarity index 90% rename from src/OpenTelemetry.Exporter.OpenTelemetryProtocol/OtlpMetricsExporter.cs rename to src/OpenTelemetry.Exporter.OpenTelemetryProtocol/OtlpMetricExporter.cs index 120acf1ad07..2272109ddc0 100644 --- a/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/OtlpMetricsExporter.cs +++ b/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/OtlpMetricExporter.cs @@ -1,4 +1,4 @@ -// +// // Copyright The OpenTelemetry Authors // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -29,25 +29,25 @@ namespace OpenTelemetry.Exporter /// the OpenTelemetry protocol (OTLP). /// [AggregationTemporality(AggregationTemporality.Cumulative | AggregationTemporality.Delta, AggregationTemporality.Cumulative)] - public class OtlpMetricsExporter : BaseOtlpExporter + public class OtlpMetricExporter : BaseOtlpExporter { private readonly OtlpCollector.MetricsService.IMetricsServiceClient metricsClient; /// - /// Initializes a new instance of the class. + /// Initializes a new instance of the class. /// /// Configuration options for the exporter. - public OtlpMetricsExporter(OtlpExporterOptions options) + public OtlpMetricExporter(OtlpExporterOptions options) : this(options, null) { } /// - /// Initializes a new instance of the class. + /// Initializes a new instance of the class. /// /// Configuration options for the exporter. /// . - internal OtlpMetricsExporter(OtlpExporterOptions options, OtlpCollector.MetricsService.IMetricsServiceClient metricsServiceClient = null) + internal OtlpMetricExporter(OtlpExporterOptions options, OtlpCollector.MetricsService.IMetricsServiceClient metricsServiceClient = null) : base(options) { if (metricsServiceClient != null) diff --git a/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/OtlpMetricExporterExtensions.cs b/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/OtlpMetricExporterExtensions.cs index e50820c91ab..a6b1e3aeeb1 100644 --- a/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/OtlpMetricExporterExtensions.cs +++ b/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/OtlpMetricExporterExtensions.cs @@ -25,7 +25,7 @@ namespace OpenTelemetry.Metrics public static class OtlpMetricExporterExtensions { /// - /// Adds to the . + /// Adds to the . /// /// builder to use. /// Exporter configuration options. @@ -52,7 +52,7 @@ private static MeterProviderBuilder AddOtlpExporter(MeterProviderBuilder builder { configure?.Invoke(options); - var metricExporter = new OtlpMetricsExporter(options); + var metricExporter = new OtlpMetricExporter(options); var metricReader = new PeriodicExportingMetricReader(metricExporter, options.MetricExportIntervalMilliseconds); return builder.AddReader(metricReader); } From 9591cb95e9b7615fe6306c2ca12ad075ca596ab0 Mon Sep 17 00:00:00 2001 From: Blake Date: Mon, 4 Oct 2021 21:42:49 +0100 Subject: [PATCH 11/39] Add basic wildcard support for listening to legacy activities. (#2444) --- src/OpenTelemetry/CHANGELOG.md | 3 + src/OpenTelemetry/Trace/TracerProviderSdk.cs | 98 +++++++++++++++---- test/Benchmarks/Trace/TraceBenchmarks.cs | 28 ++++++ .../Trace/TracerProviderSdkTest.cs | 75 ++++++++++++++ 4 files changed, 184 insertions(+), 20 deletions(-) diff --git a/src/OpenTelemetry/CHANGELOG.md b/src/OpenTelemetry/CHANGELOG.md index 0888695e472..8a42e851daa 100644 --- a/src/OpenTelemetry/CHANGELOG.md +++ b/src/OpenTelemetry/CHANGELOG.md @@ -5,6 +5,9 @@ * SDK is allocation-free on recording of measurements with upto 8 tags. +* TracerProviderBuilder.AddLegacySource now supports wildcard activity names. + ([#2183](https://github.com/open-telemetry/opentelemetry-dotnet/issues/2183)) + ## 1.2.0-alpha4 Released 2021-Sep-23 diff --git a/src/OpenTelemetry/Trace/TracerProviderSdk.cs b/src/OpenTelemetry/Trace/TracerProviderSdk.cs index f384995256b..4383f5b72e7 100644 --- a/src/OpenTelemetry/Trace/TracerProviderSdk.cs +++ b/src/OpenTelemetry/Trace/TracerProviderSdk.cs @@ -48,6 +48,18 @@ internal TracerProviderSdk( this.sampler = sampler; this.supportLegacyActivity = legacyActivityOperationNames.Count > 0; + bool legacyActivityWildcardMode = false; + Regex legacyActivityWildcardModeRegex = null; + foreach (var legacyName in legacyActivityOperationNames) + { + if (legacyName.Key.Contains('*')) + { + legacyActivityWildcardMode = true; + legacyActivityWildcardModeRegex = GetWildcardRegex(legacyActivityOperationNames.Keys); + break; + } + } + foreach (var processor in processors) { this.AddProcessor(processor); @@ -61,17 +73,27 @@ internal TracerProviderSdk( } } - var listener = new ActivityListener + var listener = new ActivityListener(); + + if (this.supportLegacyActivity) { - // Callback when Activity is started. - ActivityStarted = (activity) => + Func legacyActivityPredicate = null; + if (legacyActivityWildcardMode) + { + legacyActivityPredicate = activity => legacyActivityWildcardModeRegex.IsMatch(activity.OperationName); + } + else + { + legacyActivityPredicate = activity => legacyActivityOperationNames.ContainsKey(activity.OperationName); + } + + listener.ActivityStarted = activity => { OpenTelemetrySdkEventSource.Log.ActivityStarted(activity); - if (this.supportLegacyActivity && string.IsNullOrEmpty(activity.Source.Name)) + if (string.IsNullOrEmpty(activity.Source.Name)) { - // We have a legacy activity in hand now - if (legacyActivityOperationNames.ContainsKey(activity.OperationName)) + if (legacyActivityPredicate(activity)) { // Legacy activity matches the user configured list. // Call sampler for the legacy activity @@ -101,21 +123,16 @@ internal TracerProviderSdk( { this.processor?.OnStart(activity); } - }, + }; - // Callback when Activity is stopped. - ActivityStopped = (activity) => + listener.ActivityStopped = activity => { OpenTelemetrySdkEventSource.Log.ActivityStopped(activity); - if (this.supportLegacyActivity && string.IsNullOrEmpty(activity.Source.Name)) + if (string.IsNullOrEmpty(activity.Source.Name) && !legacyActivityPredicate(activity)) { - // We have a legacy activity in hand now - if (!legacyActivityOperationNames.ContainsKey(activity.OperationName)) - { - // Legacy activity doesn't match the user configured list. No need to proceed further. - return; - } + // Legacy activity doesn't match the user configured list. No need to proceed further. + return; } if (!activity.IsAllDataRequested) @@ -135,8 +152,43 @@ internal TracerProviderSdk( { this.processor?.OnEnd(activity); } - }, - }; + }; + } + else + { + listener.ActivityStarted = activity => + { + OpenTelemetrySdkEventSource.Log.ActivityStarted(activity); + + if (activity.IsAllDataRequested && SuppressInstrumentationScope.IncrementIfTriggered() == 0) + { + this.processor?.OnStart(activity); + } + }; + + listener.ActivityStopped = activity => + { + OpenTelemetrySdkEventSource.Log.ActivityStopped(activity); + + if (!activity.IsAllDataRequested) + { + return; + } + + // Spec says IsRecording must be false once span ends. + // https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/api.md#isrecording + // However, Activity has slightly different semantic + // than Span and we don't have strong reason to do this + // now, as Activity anyway allows read/write always. + // Intentionally commenting the following line. + // activity.IsAllDataRequested = false; + + if (SuppressInstrumentationScope.DecrementIfTriggered() == 0) + { + this.processor?.OnEnd(activity); + } + }; + } if (sampler is AlwaysOnSampler) { @@ -172,13 +224,13 @@ internal TracerProviderSdk( if (name.Contains('*')) { wildcardMode = true; + break; } } if (wildcardMode) { - var pattern = '^' + string.Join("|", from name in sources select "(?:" + Regex.Escape(name).Replace("\\*", ".*") + ')') + '$'; - var regex = new Regex(pattern, RegexOptions.Compiled | RegexOptions.IgnoreCase); + var regex = GetWildcardRegex(sources); // Function which takes ActivitySource and returns true/false to indicate if it should be subscribed to // or not. @@ -216,6 +268,12 @@ internal TracerProviderSdk( ActivitySource.AddActivityListener(listener); this.listener = listener; + + Regex GetWildcardRegex(IEnumerable collection) + { + var pattern = '^' + string.Join("|", from name in collection select "(?:" + Regex.Escape(name).Replace("\\*", ".*") + ')') + '$'; + return new Regex(pattern, RegexOptions.Compiled | RegexOptions.IgnoreCase); + } } internal Resource Resource { get; } diff --git a/test/Benchmarks/Trace/TraceBenchmarks.cs b/test/Benchmarks/Trace/TraceBenchmarks.cs index e6e5041e953..bef76949936 100644 --- a/test/Benchmarks/Trace/TraceBenchmarks.cs +++ b/test/Benchmarks/Trace/TraceBenchmarks.cs @@ -97,6 +97,18 @@ public TraceBenchmarks() .AddLegacySource("TestOperationName2") .AddProcessor(new DummyActivityProcessor()) .Build(); + + Sdk.CreateTracerProviderBuilder() + .SetSampler(new AlwaysOnSampler()) + .AddLegacySource("ExactMatch.OperationName1") + .AddProcessor(new DummyActivityProcessor()) + .Build(); + + Sdk.CreateTracerProviderBuilder() + .SetSampler(new AlwaysOnSampler()) + .AddLegacySource("WildcardMatch.*") + .AddProcessor(new DummyActivityProcessor()) + .Build(); } [Benchmark] @@ -175,6 +187,22 @@ public void TwoInstrumentations() } } + [Benchmark] + public void LegacyActivity_ExactMatchMode() + { + using (var activity = new Activity("ExactMatch.OperationName1").Start()) + { + } + } + + [Benchmark] + public void LegacyActivity_WildcardMatchMode() + { + using (var activity = new Activity("WildcardMatch.OperationName1").Start()) + { + } + } + internal class DummyActivityProcessor : BaseProcessor { } diff --git a/test/OpenTelemetry.Tests/Trace/TracerProviderSdkTest.cs b/test/OpenTelemetry.Tests/Trace/TracerProviderSdkTest.cs index 8d6c0069047..9f77a567e3b 100644 --- a/test/OpenTelemetry.Tests/Trace/TracerProviderSdkTest.cs +++ b/test/OpenTelemetry.Tests/Trace/TracerProviderSdkTest.cs @@ -949,6 +949,81 @@ public void TracerProviderSdkFlushesProcessorForcibly() Assert.True(testActivityProcessor.ForceFlushCalled); } + [Fact] + public void SdkSamplesAndProcessesLegacySourceWhenAddLegacySourceIsCalledWithWildcardValue() + { + var sampledActivities = new List(); + var sampler = new TestSampler + { + SamplingAction = + (samplingParameters) => + { + sampledActivities.Add(samplingParameters.Name); + return new SamplingResult(SamplingDecision.RecordAndSample); + }, + }; + + using TestActivityProcessor testActivityProcessor = new TestActivityProcessor(); + + var onStartProcessedActivities = new List(); + var onStopProcessedActivities = new List(); + testActivityProcessor.StartAction = + (a) => + { + Assert.Contains(a.OperationName, sampledActivities); + Assert.False(Sdk.SuppressInstrumentation); + Assert.True(a.IsAllDataRequested); // If Proccessor.OnStart is called, activity's IsAllDataRequested is set to true + onStartProcessedActivities.Add(a.OperationName); + }; + + testActivityProcessor.EndAction = + (a) => + { + Assert.False(Sdk.SuppressInstrumentation); + Assert.True(a.IsAllDataRequested); // If Processor.OnEnd is called, activity's IsAllDataRequested is set to true + onStopProcessedActivities.Add(a.OperationName); + }; + + var legacySourceNamespaces = new[] { "LegacyNamespace.*", "Namespace.*.Operation" }; + using var activitySource = new ActivitySource(ActivitySourceName); + + // AddLegacyOperationName chained to TracerProviderBuilder + using var tracerProvider = Sdk.CreateTracerProviderBuilder() + .SetSampler(sampler) + .AddProcessor(testActivityProcessor) + .AddLegacySource(legacySourceNamespaces[0]) + .AddLegacySource(legacySourceNamespaces[1]) + .AddSource(ActivitySourceName) + .Build(); + + foreach (var ns in legacySourceNamespaces) + { + var startOpName = ns.Replace("*", "Start"); + Activity startOperation = new Activity(startOpName); + startOperation.Start(); + startOperation.Stop(); + + Assert.Contains(startOpName, onStartProcessedActivities); // Processor.OnStart is called since we added a legacy OperationName + Assert.Contains(startOpName, onStopProcessedActivities); // Processor.OnEnd is called since we added a legacy OperationName + + var stopOpName = ns.Replace("*", "Stop"); + Activity stopOperation = new Activity(stopOpName); + stopOperation.Start(); + stopOperation.Stop(); + + Assert.Contains(stopOpName, onStartProcessedActivities); // Processor.OnStart is called since we added a legacy OperationName + Assert.Contains(stopOpName, onStopProcessedActivities); // Processor.OnEnd is called since we added a legacy OperationName + } + + // Ensure we can still process "normal" activities when in legacy wildcard mode. + Activity nonLegacyActivity = activitySource.StartActivity("TestActivity"); + nonLegacyActivity.Start(); + nonLegacyActivity.Stop(); + + Assert.Contains(nonLegacyActivity.OperationName, onStartProcessedActivities); // Processor.OnStart is called since we added a legacy OperationName + Assert.Contains(nonLegacyActivity.OperationName, onStopProcessedActivities); // Processor.OnEnd is called since we added a legacy OperationName + } + public void Dispose() { GC.SuppressFinalize(this); From a4b25f117d599a4ba6ea0ac49f9423353730da55 Mon Sep 17 00:00:00 2001 From: Mikel Blanchard Date: Mon, 4 Oct 2021 14:07:22 -0700 Subject: [PATCH 12/39] Prometheus: Unit tests part 2 (#2445) --- .../PrometheusExporterExtensionsTests.cs | 101 +++++++++++++++--- 1 file changed, 86 insertions(+), 15 deletions(-) diff --git a/test/OpenTelemetry.Exporter.Prometheus.Tests/PrometheusExporterExtensionsTests.cs b/test/OpenTelemetry.Exporter.Prometheus.Tests/PrometheusExporterExtensionsTests.cs index 3762ddb13aa..8527fda149c 100644 --- a/test/OpenTelemetry.Exporter.Prometheus.Tests/PrometheusExporterExtensionsTests.cs +++ b/test/OpenTelemetry.Exporter.Prometheus.Tests/PrometheusExporterExtensionsTests.cs @@ -18,6 +18,7 @@ using System.Collections.Generic; using System.Diagnostics.Metrics; using System.IO; +using System.Reflection; using System.Text; using OpenTelemetry.Metrics; using OpenTelemetry.Tests; @@ -27,30 +28,42 @@ namespace OpenTelemetry.Exporter.Prometheus.Tests { public sealed class PrometheusExporterExtensionsTests { - [Fact] - public void WriteMetricsCollectionTest() + private const string MeterName = "PrometheusExporterExtensionsTests.WriteMetricsCollectionTest.Meter"; + + [Theory] + [InlineData(nameof(WriteLongSum))] + [InlineData(nameof(WriteDoubleSum))] + [InlineData(nameof(WriteLongGauge))] + [InlineData(nameof(WriteDoubleGauge))] + [InlineData(nameof(WriteHistogram))] + public void WriteMetricsCollectionTest(string writeActionMethodName) { + using var meter = new Meter(MeterName, "0.0.1"); + + MethodInfo writeAction = typeof(PrometheusExporterExtensionsTests).GetMethod(writeActionMethodName, BindingFlags.NonPublic | BindingFlags.Static); + if (writeAction == null) + { + throw new InvalidOperationException($"Write action {writeActionMethodName} could not be found reflectively."); + } + var tags = new KeyValuePair[] { new KeyValuePair("key1", "value1"), new KeyValuePair("key2", "value2"), }; - using var provider = Sdk.CreateMeterProviderBuilder() - .AddSource("TestMeter") - .AddReader(new BaseExportingMetricReader(new TestExporter(RunTest))) - .Build(); - - using var meter = new Meter("TestMeter", "0.0.1"); - - var counter = meter.CreateCounter("counter"); - - counter.Add(100, tags); + string tagsExpected = "{key1=\"value1\",key2=\"value2\"}"; + string expected = null; var testCompleted = false; - // Invokes the TestExporter which will invoke RunTest - provider.ForceFlush(3000); + using (var provider = Sdk.CreateMeterProviderBuilder() + .AddSource(MeterName) + .AddReader(new BaseExportingMetricReader(new TestExporter(RunTest))) + .Build()) + { + expected = (string)writeAction.Invoke(null, new object[] { meter, tags, tagsExpected }); + } Assert.True(testCompleted); @@ -67,11 +80,69 @@ void RunTest(Batch metrics) } Assert.Equal( - "# TYPE counter counter\ncounter{key1=\"value1\",key2=\"value2\"} 100 1633041000000\n", + expected, Encoding.UTF8.GetString(ms.ToArray())); testCompleted = true; } } + + private static string WriteLongSum(Meter meter, KeyValuePair[] tags, string tagsExpected) + { + var counter = meter.CreateCounter("counter_int", description: "Prometheus help text goes here \n escaping."); + counter.Add(100, tags); + + return $"# HELP counter_intPrometheus help text goes here \\n escaping.\n# TYPE counter_int counter\ncounter_int{tagsExpected} 100 1633041000000\n"; + } + + private static string WriteDoubleSum(Meter meter, KeyValuePair[] tags, string tagsExpected) + { + var counter = meter.CreateCounter("counter_double"); + counter.Add(100.18D, tags); + counter.Add(0.99D, tags); + + return $"# TYPE counter_double counter\ncounter_double{tagsExpected} 101.17 1633041000000\n"; + } + + private static string WriteLongGauge(Meter meter, KeyValuePair[] tags, string tagsExpected) + { + var gauge = meter.CreateObservableGauge( + "gauge_long", + () => new Measurement[] { new Measurement(18, tags) }); + + return $"# TYPE gauge_long gauge\ngauge_long{tagsExpected} 18 1633041000000\n"; + } + + private static string WriteDoubleGauge(Meter meter, KeyValuePair[] tags, string tagsExpected) + { + var value = 0.18F; + + var gauge = meter.CreateObservableGauge( + "gauge_double", + () => new Measurement[] { new Measurement(99F, tags), new Measurement(value, tags) }); + + return $"# TYPE gauge_double gauge\ngauge_double{tagsExpected} {(double)value} 1633041000000\n"; + } + + private static string WriteHistogram(Meter meter, KeyValuePair[] tags, string tagsExpected) + { + var histogram = meter.CreateHistogram("histogram_name"); + histogram.Record(100, tags); + histogram.Record(18, tags); + + return "# TYPE histogram_name histogram\nhistogram_name_sum{key1=\"value1\",key2=\"value2\"} 118 1633041000000\n" + + "histogram_name_count{key1=\"value1\",key2=\"value2\"} 2 1633041000000\n" + + "histogram_name_bucket{key1=\"value1\",key2=\"value2\",le=\"0\"} 0 1633041000000\n" + + "histogram_name_bucket{key1=\"value1\",key2=\"value2\",le=\"5\"} 0 1633041000000\n" + + "histogram_name_bucket{key1=\"value1\",key2=\"value2\",le=\"10\"} 0 1633041000000\n" + + "histogram_name_bucket{key1=\"value1\",key2=\"value2\",le=\"25\"} 1 1633041000000\n" + + "histogram_name_bucket{key1=\"value1\",key2=\"value2\",le=\"50\"} 1 1633041000000\n" + + "histogram_name_bucket{key1=\"value1\",key2=\"value2\",le=\"75\"} 1 1633041000000\n" + + "histogram_name_bucket{key1=\"value1\",key2=\"value2\",le=\"100\"} 2 1633041000000\n" + + "histogram_name_bucket{key1=\"value1\",key2=\"value2\",le=\"250\"} 2 1633041000000\n" + + "histogram_name_bucket{key1=\"value1\",key2=\"value2\",le=\"500\"} 2 1633041000000\n" + + "histogram_name_bucket{key1=\"value1\",key2=\"value2\",le=\"1000\"} 2 1633041000000\n" + + "histogram_name_bucket{key1=\"value1\",key2=\"value2\",le=\"+Inf\"} 2 1633041000000\n"; + } } } From 2cb0aa8dd82c08edc813078fe728d4294970537b Mon Sep 17 00:00:00 2001 From: Cijo Thomas Date: Mon, 4 Oct 2021 23:22:14 -0700 Subject: [PATCH 13/39] Benchmark and stress test improvements (#2447) --- .../Metrics/MetricsViewBenchmarks.cs | 124 ++++++++++++++++++ test/OpenTelemetry.Tests.Stress/Skeleton.cs | 12 +- 2 files changed, 135 insertions(+), 1 deletion(-) create mode 100644 test/Benchmarks/Metrics/MetricsViewBenchmarks.cs diff --git a/test/Benchmarks/Metrics/MetricsViewBenchmarks.cs b/test/Benchmarks/Metrics/MetricsViewBenchmarks.cs new file mode 100644 index 00000000000..71f6f0c431e --- /dev/null +++ b/test/Benchmarks/Metrics/MetricsViewBenchmarks.cs @@ -0,0 +1,124 @@ +// +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +using System; +using System.Collections.Generic; +using System.Diagnostics.Metrics; +using System.Threading; +using BenchmarkDotNet.Attributes; +using OpenTelemetry; +using OpenTelemetry.Metrics; + +/* +BenchmarkDotNet=v0.12.1, OS=Windows 10.0.19043 +Intel Core i7-8650U CPU 1.90GHz (Kaby Lake R), 1 CPU, 8 logical and 4 physical cores +.NET Core SDK=5.0.401 + [Host] : .NET Core 5.0.10 (CoreCLR 5.0.1021.41214, CoreFX 5.0.1021.41214), X64 RyuJIT + DefaultJob : .NET Core 5.0.10 (CoreCLR 5.0.1021.41214, CoreFX 5.0.1021.41214), X64 RyuJIT + + +| Method | ViewConfig | Mean | Error | StdDev | Gen 0 | Gen 1 | Gen 2 | Allocated | +|----------------------------------------- |--------------------- |---------:|---------:|---------:|------:|------:|------:|----------:| +| CounterMeasurementRecordingWithThreeTags | NoView | 503.2 ns | 8.36 ns | 7.82 ns | - | - | - | - | +| CounterMeasurementRecordingWithThreeTags | ViewNoInstrSelect | 552.7 ns | 10.98 ns | 19.24 ns | - | - | - | - | +| CounterMeasurementRecordingWithThreeTags | ViewSelectsInstr | 556.0 ns | 11.12 ns | 24.18 ns | - | - | - | - | +*/ + +namespace Benchmarks.Metrics +{ + [MemoryDiagnoser] + public class MetricsViewBenchmarks + { + private static readonly ThreadLocal ThreadLocalRandom = new ThreadLocal(() => new Random()); + private static readonly string[] DimensionValues = new string[] { "DimVal1", "DimVal2", "DimVal3", "DimVal4", "DimVal5", "DimVal6", "DimVal7", "DimVal8", "DimVal9", "DimVal10" }; + private static readonly int DimensionsValuesLength = DimensionValues.Length; + private Counter counter; + private MeterProvider provider; + private Meter meter; + + public enum ViewConfiguration + { + /// + /// No views registered in the provider. + /// + NoView, + + /// + /// Provider has view registered, but it doesn't select the instrument. + /// This tests the perf impact View has on hot path, for those + /// instruments not participating in View feature. + /// + ViewNoInstrSelect, + + /// + /// Provider has view registered and it does select the instrument. + /// + ViewSelectsInstr, + } + + [Params( + ViewConfiguration.NoView, + ViewConfiguration.ViewNoInstrSelect, + ViewConfiguration.ViewSelectsInstr)] + public ViewConfiguration ViewConfig { get; set; } + + [GlobalSetup] + public void Setup() + { + this.meter = new Meter("TestMeter"); + this.counter = this.meter.CreateCounter("counter"); + + if (this.ViewConfig == ViewConfiguration.NoView) + { + this.provider = Sdk.CreateMeterProviderBuilder() + .AddSource(this.meter.Name) + .Build(); + } + else if (this.ViewConfig == ViewConfiguration.ViewNoInstrSelect) + { + this.provider = Sdk.CreateMeterProviderBuilder() + .AddSource(this.meter.Name) + .AddView("nomatch", new MetricStreamConfiguration() { TagKeys = new string[] { "DimName1", "DimName2", "DimName3" } }) + .Build(); + } + else if (this.ViewConfig == ViewConfiguration.ViewSelectsInstr) + { + this.provider = Sdk.CreateMeterProviderBuilder() + .AddSource(this.meter.Name) + .AddView(this.counter.Name, new MetricStreamConfiguration() { TagKeys = new string[] { "DimName1", "DimName2", "DimName3" } }) + .Build(); + } + } + + [GlobalCleanup] + public void Cleanup() + { + this.meter?.Dispose(); + this.provider?.Dispose(); + } + + [Benchmark] + public void CounterMeasurementRecordingWithThreeTags() + { + var random = ThreadLocalRandom.Value; + this.counter?.Add( + 100, + new KeyValuePair("DimName1", DimensionValues[random.Next(0, DimensionsValuesLength)]), + new KeyValuePair("DimName2", DimensionValues[random.Next(0, DimensionsValuesLength)]), + new KeyValuePair("DimName3", DimensionValues[random.Next(0, DimensionsValuesLength)])); + } + } +} diff --git a/test/OpenTelemetry.Tests.Stress/Skeleton.cs b/test/OpenTelemetry.Tests.Stress/Skeleton.cs index d8a3bf4dcc5..b1b819439c3 100644 --- a/test/OpenTelemetry.Tests.Stress/Skeleton.cs +++ b/test/OpenTelemetry.Tests.Stress/Skeleton.cs @@ -46,6 +46,8 @@ public static void Stress(int concurrency = 0) } var statistics = new long[concurrency]; + var watchForTotal = new Stopwatch(); + watchForTotal.Start(); Parallel.Invoke( () => @@ -112,7 +114,15 @@ public static void Stress(int concurrency = 0) }); }); - Console.WriteLine(output); + watchForTotal.Stop(); + var cntLoopsTotal = (ulong)statistics.Sum(); + var totalLoopsPerSecond = (double)cntLoopsTotal / ((double)watchForTotal.ElapsedMilliseconds / 1000.0); + var cntCpuCyclesTotal = GetCpuCycles(); + var cpuCyclesPerLoopTotal = cntLoopsTotal == 0 ? 0 : cntCpuCyclesTotal / cntLoopsTotal; + Console.WriteLine("Stopping the stress test..."); + Console.WriteLine($"* Total Loops: {cntLoopsTotal:n0}"); + Console.WriteLine($"* Average Loops/Second: {totalLoopsPerSecond:n0}"); + Console.WriteLine($"* Average CPU Cycles/Loop: {cpuCyclesPerLoopTotal:n0}"); } [DllImport("kernel32.dll")] From 2512aa170941ca2739200fbfe040e150e42e088d Mon Sep 17 00:00:00 2001 From: Cijo Thomas Date: Tue, 5 Oct 2021 10:18:21 -0700 Subject: [PATCH 14/39] View Part 3 - Support for configuring tagkeys to be used for aggregation (#2442) --- src/OpenTelemetry/Metrics/AggregatorStore.cs | 209 ++++++++++++++---- src/OpenTelemetry/Metrics/MeterProviderSdk.cs | 13 +- src/OpenTelemetry/Metrics/Metric.cs | 9 +- .../Metrics/ThreadStaticStorage.cs | 53 +++++ .../Metrics/MetricViewTests.cs | 64 ++++++ 5 files changed, 286 insertions(+), 62 deletions(-) diff --git a/src/OpenTelemetry/Metrics/AggregatorStore.cs b/src/OpenTelemetry/Metrics/AggregatorStore.cs index 8bccd5ef3e7..e151875b3a9 100644 --- a/src/OpenTelemetry/Metrics/AggregatorStore.cs +++ b/src/OpenTelemetry/Metrics/AggregatorStore.cs @@ -27,6 +27,8 @@ internal class AggregatorStore internal const int MaxMetricPoints = 2000; private static readonly ObjectArrayEqualityComparer ObjectArrayComparer = new ObjectArrayEqualityComparer(); private readonly object lockZeroTags = new object(); + private readonly HashSet tagKeysInteresting; + private readonly int tagsKeysInterestingCount; // Two-Level lookup. TagKeys x [ TagValues x Metrics ] private readonly ConcurrentDictionary> keyValue2MetricAggs = @@ -40,73 +42,52 @@ internal class AggregatorStore private double[] histogramBounds; private DateTimeOffset startTimeExclusive; private DateTimeOffset endTimeInclusive; + private UpdateLongDelegate updateLongCallback; + private UpdateDoubleDelegate updateDoubleCallback; - internal AggregatorStore(AggregationType aggType, AggregationTemporality temporality, double[] histogramBounds) + internal AggregatorStore( + AggregationType aggType, + AggregationTemporality temporality, + double[] histogramBounds, + string[] tagKeysInteresting = null) { this.metrics = new MetricPoint[MaxMetricPoints]; this.aggType = aggType; this.temporality = temporality; this.histogramBounds = histogramBounds; this.startTimeExclusive = DateTimeOffset.UtcNow; - } - - internal int FindMetricAggregators(ReadOnlySpan> tags) - { - int tagLength = tags.Length; - if (tagLength == 0) - { - this.InitializeZeroTagPointIfNotInitialized(); - return 0; - } - - var storage = ThreadStaticStorage.GetStorage(); - - storage.SplitToKeysAndValues(tags, tagLength, out var tagKey, out var tagValue); - - if (tagLength > 1) + if (tagKeysInteresting == null) { - Array.Sort(tagKey, tagValue); + this.updateLongCallback = this.UpdateLong; + this.updateDoubleCallback = this.UpdateDouble; } - - return this.LookupAggregatorStore(tagKey, tagValue, tagLength); - } - - internal void UpdateLong(long value, ReadOnlySpan> tags) - { - try + else { - var index = this.FindMetricAggregators(tags); - if (index < 0) + this.updateLongCallback = this.UpdateLongCustomTags; + this.updateDoubleCallback = this.UpdateDoubleCustomTags; + var hs = new HashSet(StringComparer.Ordinal); + foreach (var key in tagKeysInteresting) { - // TODO: Measurement dropped due to MemoryPoint cap hit. - return; + hs.Add(key); } - this.metrics[index].Update(value); - } - catch (Exception) - { - // TODO: Measurement dropped due to internal exception. + this.tagKeysInteresting = hs; + this.tagsKeysInterestingCount = hs.Count; } } - internal void UpdateDouble(double value, ReadOnlySpan> tags) + private delegate void UpdateLongDelegate(long value, ReadOnlySpan> tags); + + private delegate void UpdateDoubleDelegate(double value, ReadOnlySpan> tags); + + internal void Update(long value, ReadOnlySpan> tags) { - try - { - var index = this.FindMetricAggregators(tags); - if (index < 0) - { - // TODO: Measurement dropped due to MemoryPoint cap hit. - return; - } + this.updateLongCallback(value, tags); + } - this.metrics[index].Update(value); - } - catch (Exception) - { - // TODO: Measurement dropped due to internal exception. - } + internal void Update(double value, ReadOnlySpan> tags) + { + this.updateDoubleCallback(value, tags); } internal void SnapShot() @@ -233,5 +214,135 @@ private int LookupAggregatorStore(string[] tagKey, object[] tagValue, int length return aggregatorIndex; } + + private void UpdateLong(long value, ReadOnlySpan> tags) + { + try + { + var index = this.FindMetricAggregatorsDefault(tags); + if (index < 0) + { + // TODO: Measurement dropped due to MemoryPoint cap hit. + return; + } + + this.metrics[index].Update(value); + } + catch (Exception) + { + // TODO: Measurement dropped due to internal exception. + } + } + + private void UpdateLongCustomTags(long value, ReadOnlySpan> tags) + { + try + { + var index = this.FindMetricAggregatorsCustomTag(tags); + if (index < 0) + { + // TODO: Measurement dropped due to MemoryPoint cap hit. + return; + } + + this.metrics[index].Update(value); + } + catch (Exception) + { + // TODO: Measurement dropped due to internal exception. + } + } + + private void UpdateDouble(double value, ReadOnlySpan> tags) + { + try + { + var index = this.FindMetricAggregatorsDefault(tags); + if (index < 0) + { + // TODO: Measurement dropped due to MemoryPoint cap hit. + return; + } + + this.metrics[index].Update(value); + } + catch (Exception) + { + // TODO: Measurement dropped due to internal exception. + } + } + + private void UpdateDoubleCustomTags(double value, ReadOnlySpan> tags) + { + try + { + var index = this.FindMetricAggregatorsCustomTag(tags); + if (index < 0) + { + // TODO: Measurement dropped due to MemoryPoint cap hit. + return; + } + + this.metrics[index].Update(value); + } + catch (Exception) + { + // TODO: Measurement dropped due to internal exception. + } + } + + private int FindMetricAggregatorsDefault(ReadOnlySpan> tags) + { + int tagLength = tags.Length; + if (tagLength == 0) + { + this.InitializeZeroTagPointIfNotInitialized(); + return 0; + } + + var storage = ThreadStaticStorage.GetStorage(); + + storage.SplitToKeysAndValues(tags, tagLength, out var tagKey, out var tagValue); + + if (tagLength > 1) + { + Array.Sort(tagKey, tagValue); + } + + return this.LookupAggregatorStore(tagKey, tagValue, tagLength); + } + + private int FindMetricAggregatorsCustomTag(ReadOnlySpan> tags) + { + int tagLength = tags.Length; + if (tagLength == 0 || this.tagsKeysInterestingCount == 0) + { + this.InitializeZeroTagPointIfNotInitialized(); + return 0; + } + + // TODO: Get only interesting tags + // from the incoming tags + + var storage = ThreadStaticStorage.GetStorage(); + + storage.SplitToKeysAndValues(tags, tagLength, this.tagKeysInteresting, out var tagKey, out var tagValue, out var actualLength); + + // Actual number of tags depend on how many + // of the incoming tags has user opted to + // select. + if (actualLength == 0) + { + this.InitializeZeroTagPointIfNotInitialized(); + return 0; + } + + if (actualLength > 1) + { + Array.Sort(tagKey, tagValue); + } + + return this.LookupAggregatorStore(tagKey, tagValue, actualLength); + } } } diff --git a/src/OpenTelemetry/Metrics/MeterProviderSdk.cs b/src/OpenTelemetry/Metrics/MeterProviderSdk.cs index 3498aa4b308..92352ff04f1 100644 --- a/src/OpenTelemetry/Metrics/MeterProviderSdk.cs +++ b/src/OpenTelemetry/Metrics/MeterProviderSdk.cs @@ -155,15 +155,10 @@ internal MeterProviderSdk( { Metric metric; var metricDescription = metricStreamConfig?.Description ?? instrument.Description; - if (metricStreamConfig is HistogramConfiguration histogramConfig - && histogramConfig.BucketBounds != null) - { - metric = new Metric(instrument, temporality, metricStreamName, metricDescription, histogramConfig.BucketBounds); - } - else - { - metric = new Metric(instrument, temporality, metricStreamName, metricDescription); - } + string[] tagKeysInteresting = metricStreamConfig?.TagKeys; + double[] histogramBucketBounds = (metricStreamConfig is HistogramConfiguration histogramConfig + && histogramConfig.BucketBounds != null) ? histogramConfig.BucketBounds : null; + metric = new Metric(instrument, temporality, metricStreamName, metricDescription, histogramBucketBounds, tagKeysInteresting); this.metrics[index] = metric; metrics.Add(metric); diff --git a/src/OpenTelemetry/Metrics/Metric.cs b/src/OpenTelemetry/Metrics/Metric.cs index a05402e4e9c..bf9b16288fe 100644 --- a/src/OpenTelemetry/Metrics/Metric.cs +++ b/src/OpenTelemetry/Metrics/Metric.cs @@ -30,7 +30,8 @@ internal Metric( AggregationTemporality temporality, string metricName, string metricDescription, - double[] histogramBounds = null) + double[] histogramBounds = null, + string[] tagKeysInteresting = null) { this.Name = metricName; this.Description = metricDescription; @@ -103,7 +104,7 @@ internal Metric( // TODO: Log and assign some invalid Enum. } - this.aggStore = new AggregatorStore(aggType, temporality, histogramBounds ?? DefaultHistogramBounds); + this.aggStore = new AggregatorStore(aggType, temporality, histogramBounds ?? DefaultHistogramBounds, tagKeysInteresting); this.Temporality = temporality; } @@ -126,12 +127,12 @@ public BatchMetricPoint GetMetricPoints() internal void UpdateLong(long value, ReadOnlySpan> tags) { - this.aggStore.UpdateLong(value, tags); + this.aggStore.Update(value, tags); } internal void UpdateDouble(double value, ReadOnlySpan> tags) { - this.aggStore.UpdateDouble(value, tags); + this.aggStore.Update(value, tags); } internal void SnapShot() diff --git a/src/OpenTelemetry/Metrics/ThreadStaticStorage.cs b/src/OpenTelemetry/Metrics/ThreadStaticStorage.cs index 3e2dd0be834..c39485e3387 100644 --- a/src/OpenTelemetry/Metrics/ThreadStaticStorage.cs +++ b/src/OpenTelemetry/Metrics/ThreadStaticStorage.cs @@ -69,6 +69,59 @@ internal void SplitToKeysAndValues(ReadOnlySpan> ta } } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal void SplitToKeysAndValues(ReadOnlySpan> tags, int tagLength, HashSet tagKeysInteresting, out string[] tagKeys, out object[] tagValues, out int actualLength) + { + // Iterate over tags to find the exact length. + int i = 0; + for (var n = 0; n < tagLength; n++) + { + if (tagKeysInteresting.Contains(tags[n].Key)) + { + i++; + } + } + + actualLength = i; + + if (actualLength == 0) + { + tagKeys = null; + tagValues = null; + } + else if (actualLength <= MaxTagCacheSize) + { + tagKeys = this.tagStorage[actualLength - 1].TagKey; + tagValues = this.tagStorage[actualLength - 1].TagValue; + } + else + { + tagKeys = new string[actualLength]; + tagValues = new object[actualLength]; + } + + // Iterate again (!) to assign the actual value. + // TODO: The dual iteration over tags might be + // avoidable if we change the tagKey and tagObject + // to be a different type (eg: List). + // It might lead to some wasted memory. + // Also, it requires changes to the Dictionary + // used for lookup. + // The TODO here is to make that change + // separately, after benchmarking. + i = 0; + for (var n = 0; n < tagLength; n++) + { + var tag = tags[n]; + if (tagKeysInteresting.Contains(tag.Key)) + { + tagKeys[i] = tag.Key; + tagValues[i] = tag.Value; + i++; + } + } + } + internal class TagStorage { // Used to split into Key sequence, Value sequence. diff --git a/test/OpenTelemetry.Tests/Metrics/MetricViewTests.cs b/test/OpenTelemetry.Tests/Metrics/MetricViewTests.cs index bb5510e213d..606e40105c9 100644 --- a/test/OpenTelemetry.Tests/Metrics/MetricViewTests.cs +++ b/test/OpenTelemetry.Tests/Metrics/MetricViewTests.cs @@ -22,6 +22,7 @@ namespace OpenTelemetry.Metrics.Tests { +#pragma warning disable SA1000 // KeywordsMustBeSpacedCorrectly https://github.com/DotNetAnalyzers/StyleCopAnalyzers/issues/3214 public class MetricViewTests { private const int MaxTimeToAllowForFlush = 10000; @@ -229,5 +230,68 @@ public void ViewToProduceCustomHistogramBound() Assert.Equal(2, histogramPoint.BucketCounts[1]); Assert.Equal(0, histogramPoint.BucketCounts[2]); } + + [Fact] + public void ViewToSelectTagKeys() + { + using var meter1 = new Meter("ViewToSelectTagKeysTest"); + var exportedItems = new List(); + using var meterProvider = Sdk.CreateMeterProviderBuilder() + .AddSource(meter1.Name) + .AddView("FruitCounter", new MetricStreamConfiguration() + { TagKeys = new string[] { "name" }, Name = "NameOnly" }) + .AddView("FruitCounter", new MetricStreamConfiguration() + { TagKeys = new string[] { "size" }, Name = "SizeOnly" }) + .AddView("FruitCounter", new MetricStreamConfiguration() + { TagKeys = new string[] { }, Name = "NoTags" }) + .AddInMemoryExporter(exportedItems) + .Build(); + + var counter = meter1.CreateCounter("FruitCounter"); + counter.Add(10, new("name", "apple"), new("color", "red"), new("size", "small")); + counter.Add(10, new("name", "apple"), new("color", "red"), new("size", "small")); + + counter.Add(10, new("name", "apple"), new("color", "red"), new("size", "medium")); + counter.Add(10, new("name", "apple"), new("color", "red"), new("size", "medium")); + + counter.Add(10, new("name", "apple"), new("color", "red"), new("size", "large")); + counter.Add(10, new("name", "apple"), new("color", "red"), new("size", "large")); + + meterProvider.ForceFlush(MaxTimeToAllowForFlush); + Assert.Equal(3, exportedItems.Count); + var metric = exportedItems[0]; + Assert.Equal("NameOnly", metric.Name); + List metricPoints = new List(); + foreach (ref var mp in metric.GetMetricPoints()) + { + metricPoints.Add(mp); + } + + // Only one point expected "apple" + Assert.Single(metricPoints); + + metric = exportedItems[1]; + Assert.Equal("SizeOnly", metric.Name); + metricPoints.Clear(); + foreach (ref var mp in metric.GetMetricPoints()) + { + metricPoints.Add(mp); + } + + // 3 points small,medium,large expected + Assert.Equal(3, metricPoints.Count); + + metric = exportedItems[2]; + Assert.Equal("NoTags", metric.Name); + metricPoints.Clear(); + foreach (ref var mp in metric.GetMetricPoints()) + { + metricPoints.Add(mp); + } + + // Single point expected. + Assert.Single(metricPoints); + } } +#pragma warning restore SA1000 // KeywordsMustBeSpacedCorrectly } From 1aa4da209802469908c24d78f0f9812f1f917dbb Mon Sep 17 00:00:00 2001 From: Oleksiy Dubinin <88040756+rypdal@users.noreply.github.com> Date: Tue, 5 Oct 2021 20:35:54 +0200 Subject: [PATCH 15/39] Issue/2292 otlp http binary protobuf trace exporter (#2316) --- examples/Console/Program.cs | 8 +- examples/Console/TestOtlpExporter.cs | 28 ++- .../.publicApi/net461/PublicAPI.Unshipped.txt | 5 + .../netstandard2.0/PublicAPI.Unshipped.txt | 5 + .../netstandard2.1/PublicAPI.Unshipped.txt | 5 + .../CHANGELOG.md | 8 +- .../ExportClient/BaseOtlpGrpcExportClient.cs} | 46 ++-- .../ExportClient/BaseOtlpHttpExportClient.cs | 87 ++++++++ .../ExportClient/IExportClient.cs | 45 ++++ .../OtlpGrpcMetricsExportClient.cs | 62 ++++++ .../ExportClient/OtlpGrpcTraceExportClient.cs | 62 ++++++ .../ExportClient/OtlpHttpTraceExportClient.cs | 60 ++++++ ...penTelemetryProtocolExporterEventSource.cs | 6 + ...etry.Exporter.OpenTelemetryProtocol.csproj | 14 +- .../OtlpExportProtocol.cs | 34 +++ .../OtlpExporterOptions.cs | 23 ++ .../OtlpExporterOptionsExtensions.cs | 133 ++++++++++++ .../OtlpExporterOptionsGrpcExtensions.cs | 82 ------- .../OtlpMetricExporter.cs | 54 ++--- .../OtlpTraceExporter.cs | 48 +++-- .../README.md | 12 +- .../Exporter/OtlpExporterBenchmarks.cs | 7 +- .../OtlpHttpTraceExportClientTests.cs | 204 ++++++++++++++++++ .../OtlpExporterOptionsExtensionsTests.cs | 135 ++++++++++++ .../OtlpExporterOptionsGrpcExtensionsTests.cs | 67 ------ .../OtlpExporterOptionsTests.cs | 18 ++ .../OtlpMetricsExporterTests.cs | 1 + .../OtlpTraceExporterTests.cs | 20 +- 28 files changed, 1027 insertions(+), 252 deletions(-) rename src/OpenTelemetry.Exporter.OpenTelemetryProtocol/{BaseOtlpExporter.cs => Implementation/ExportClient/BaseOtlpGrpcExportClient.cs} (56%) create mode 100644 src/OpenTelemetry.Exporter.OpenTelemetryProtocol/Implementation/ExportClient/BaseOtlpHttpExportClient.cs create mode 100644 src/OpenTelemetry.Exporter.OpenTelemetryProtocol/Implementation/ExportClient/IExportClient.cs create mode 100644 src/OpenTelemetry.Exporter.OpenTelemetryProtocol/Implementation/ExportClient/OtlpGrpcMetricsExportClient.cs create mode 100644 src/OpenTelemetry.Exporter.OpenTelemetryProtocol/Implementation/ExportClient/OtlpGrpcTraceExportClient.cs create mode 100644 src/OpenTelemetry.Exporter.OpenTelemetryProtocol/Implementation/ExportClient/OtlpHttpTraceExportClient.cs create mode 100644 src/OpenTelemetry.Exporter.OpenTelemetryProtocol/OtlpExportProtocol.cs create mode 100644 src/OpenTelemetry.Exporter.OpenTelemetryProtocol/OtlpExporterOptionsExtensions.cs delete mode 100644 src/OpenTelemetry.Exporter.OpenTelemetryProtocol/OtlpExporterOptionsGrpcExtensions.cs create mode 100644 test/OpenTelemetry.Exporter.OpenTelemetryProtocol.Tests/Implementation/ExportClient/OtlpHttpTraceExportClientTests.cs create mode 100644 test/OpenTelemetry.Exporter.OpenTelemetryProtocol.Tests/OtlpExporterOptionsExtensionsTests.cs delete mode 100644 test/OpenTelemetry.Exporter.OpenTelemetryProtocol.Tests/OtlpExporterOptionsGrpcExtensionsTests.cs diff --git a/examples/Console/Program.cs b/examples/Console/Program.cs index dbba17e57f8..7a40dbe168e 100644 --- a/examples/Console/Program.cs +++ b/examples/Console/Program.cs @@ -15,6 +15,7 @@ // using CommandLine; +using OpenTelemetry.Exporter; namespace Examples.Console { @@ -32,7 +33,7 @@ public class Program /// dotnet run -p Examples.Console.csproj zipkin -u http://localhost:9411/api/v2/spans /// dotnet run -p Examples.Console.csproj jaeger -h localhost -p 6831 /// dotnet run -p Examples.Console.csproj prometheus -p 9184 -d 2 - /// dotnet run -p Examples.Console.csproj otlp -e "http://localhost:4317" + /// dotnet run -p Examples.Console.csproj otlp -e "http://localhost:4317" -p "grpc" /// dotnet run -p Examples.Console.csproj zpages /// dotnet run -p Examples.Console.csproj metrics --help /// @@ -55,7 +56,7 @@ public static void Main(string[] args) (ConsoleOptions options) => TestConsoleExporter.Run(options), (OpenTelemetryShimOptions options) => TestOTelShimWithConsoleExporter.Run(options), (OpenTracingShimOptions options) => TestOpenTracingShim.Run(options), - (OtlpOptions options) => TestOtlpExporter.Run(options.Endpoint), + (OtlpOptions options) => TestOtlpExporter.Run(options.Endpoint, options.Protocol), (InMemoryOptions options) => TestInMemoryExporter.Run(options), errs => 1); } @@ -163,6 +164,9 @@ internal class OtlpOptions { [Option('e', "endpoint", HelpText = "Target to which the exporter is going to send traces or metrics", Default = "http://localhost:4317")] public string Endpoint { get; set; } + + [Option('p', "protocol", HelpText = "Transport protocol used by exporter. Supported values: grpc and http/protobuf.", Default = "grpc")] + public string Protocol { get; set; } } [Verb("inmemory", HelpText = "Specify the options required to test InMemory Exporter")] diff --git a/examples/Console/TestOtlpExporter.cs b/examples/Console/TestOtlpExporter.cs index 7002f2a69b6..da28553ade4 100644 --- a/examples/Console/TestOtlpExporter.cs +++ b/examples/Console/TestOtlpExporter.cs @@ -16,6 +16,7 @@ using System; using OpenTelemetry; +using OpenTelemetry.Exporter; using OpenTelemetry.Resources; using OpenTelemetry.Trace; @@ -23,7 +24,7 @@ namespace Examples.Console { internal static class TestOtlpExporter { - internal static object Run(string endpoint) + internal static object Run(string endpoint, string protocol) { /* * Prerequisite to run this example: @@ -49,22 +50,33 @@ internal static object Run(string endpoint) * For more information about the OpenTelemetry Collector go to https://github.com/open-telemetry/opentelemetry-collector * */ - return RunWithActivitySource(endpoint); + return RunWithActivitySource(endpoint, protocol); } - private static object RunWithActivitySource(string endpoint) + private static object RunWithActivitySource(string endpoint, string protocol) { // Adding the OtlpExporter creates a GrpcChannel. // This switch must be set before creating a GrpcChannel/HttpClient when calling an insecure gRPC service. // See: https://docs.microsoft.com/aspnet/core/grpc/troubleshoot#call-insecure-grpc-services-with-net-core-client AppContext.SetSwitch("System.Net.Http.SocketsHttpHandler.Http2UnencryptedSupport", true); + var otlpExportProtocol = ToOtlpExportProtocol(protocol); + if (!otlpExportProtocol.HasValue) + { + System.Console.WriteLine($"Export protocol {protocol} is not supported. Default protocol 'grpc' will be used."); + otlpExportProtocol = OtlpExportProtocol.Grpc; + } + // Enable OpenTelemetry for the sources "Samples.SampleServer" and "Samples.SampleClient" // and use OTLP exporter. using var openTelemetry = Sdk.CreateTracerProviderBuilder() .AddSource("Samples.SampleClient", "Samples.SampleServer") .SetResourceBuilder(ResourceBuilder.CreateDefault().AddService("otlp-test")) - .AddOtlpExporter(opt => opt.Endpoint = new Uri(endpoint)) + .AddOtlpExporter(opt => + { + opt.Endpoint = new Uri(endpoint); + opt.Protocol = otlpExportProtocol.Value; + }) .Build(); // The above line is required only in Applications @@ -81,5 +93,13 @@ private static object RunWithActivitySource(string endpoint) return null; } + + private static OtlpExportProtocol? ToOtlpExportProtocol(string protocol) => + protocol.Trim().ToLower() switch + { + "grpc" => OtlpExportProtocol.Grpc, + "http/protobuf" => OtlpExportProtocol.HttpProtobuf, + _ => null + }; } } diff --git a/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/.publicApi/net461/PublicAPI.Unshipped.txt b/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/.publicApi/net461/PublicAPI.Unshipped.txt index b457c2d381d..ef0a2267389 100644 --- a/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/.publicApi/net461/PublicAPI.Unshipped.txt +++ b/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/.publicApi/net461/PublicAPI.Unshipped.txt @@ -10,9 +10,14 @@ OpenTelemetry.Exporter.OtlpExporterOptions.Headers.set -> void OpenTelemetry.Exporter.OtlpExporterOptions.OtlpExporterOptions() -> void OpenTelemetry.Exporter.OtlpExporterOptions.TimeoutMilliseconds.get -> int OpenTelemetry.Exporter.OtlpExporterOptions.TimeoutMilliseconds.set -> void +OpenTelemetry.Exporter.OtlpExporterOptions.Protocol.get -> OpenTelemetry.Exporter.ExportProtocol +OpenTelemetry.Exporter.OtlpExporterOptions.Protocol.set -> void OpenTelemetry.Exporter.OtlpTraceExporter OpenTelemetry.Exporter.OtlpTraceExporter.OtlpTraceExporter(OpenTelemetry.Exporter.OtlpExporterOptions options) -> void OpenTelemetry.Trace.OtlpTraceExporterHelperExtensions override OpenTelemetry.Exporter.OtlpTraceExporter.Export(in OpenTelemetry.Batch activityBatch) -> OpenTelemetry.ExportResult override OpenTelemetry.Exporter.OtlpTraceExporter.OnShutdown(int timeoutMilliseconds) -> bool static OpenTelemetry.Trace.OtlpTraceExporterHelperExtensions.AddOtlpExporter(this OpenTelemetry.Trace.TracerProviderBuilder builder, System.Action configure = null) -> OpenTelemetry.Trace.TracerProviderBuilder +OpenTelemetry.Exporter.OtlpExportProtocol +OpenTelemetry.Exporter.OtlpExportProtocol.Grpc = 0 -> OpenTelemetry.Exporter.OtlpExportProtocol +OpenTelemetry.Exporter.OtlpExportProtocol.HttpProtobuf = 1 -> OpenTelemetry.Exporter.OtlpExportProtocol diff --git a/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/.publicApi/netstandard2.0/PublicAPI.Unshipped.txt b/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/.publicApi/netstandard2.0/PublicAPI.Unshipped.txt index e69de29bb2d..b5d2629ac07 100644 --- a/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/.publicApi/netstandard2.0/PublicAPI.Unshipped.txt +++ b/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/.publicApi/netstandard2.0/PublicAPI.Unshipped.txt @@ -0,0 +1,5 @@ +OpenTelemetry.Exporter.OtlpExporterOptions.Protocol.get -> OpenTelemetry.Exporter.ExportProtocol +OpenTelemetry.Exporter.OtlpExporterOptions.Protocol.set -> void +OpenTelemetry.Exporter.OtlpExportProtocol +OpenTelemetry.Exporter.OtlpExportProtocol.Grpc = 0 -> OpenTelemetry.Exporter.OtlpExportProtocol +OpenTelemetry.Exporter.OtlpExportProtocol.HttpProtobuf = 1 -> OpenTelemetry.Exporter.OtlpExportProtocol diff --git a/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/.publicApi/netstandard2.1/PublicAPI.Unshipped.txt b/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/.publicApi/netstandard2.1/PublicAPI.Unshipped.txt index e69de29bb2d..b5d2629ac07 100644 --- a/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/.publicApi/netstandard2.1/PublicAPI.Unshipped.txt +++ b/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/.publicApi/netstandard2.1/PublicAPI.Unshipped.txt @@ -0,0 +1,5 @@ +OpenTelemetry.Exporter.OtlpExporterOptions.Protocol.get -> OpenTelemetry.Exporter.ExportProtocol +OpenTelemetry.Exporter.OtlpExporterOptions.Protocol.set -> void +OpenTelemetry.Exporter.OtlpExportProtocol +OpenTelemetry.Exporter.OtlpExportProtocol.Grpc = 0 -> OpenTelemetry.Exporter.OtlpExportProtocol +OpenTelemetry.Exporter.OtlpExportProtocol.HttpProtobuf = 1 -> OpenTelemetry.Exporter.OtlpExportProtocol diff --git a/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/CHANGELOG.md b/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/CHANGELOG.md index eb99a4b86d6..675e9209afb 100644 --- a/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/CHANGELOG.md +++ b/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/CHANGELOG.md @@ -5,6 +5,10 @@ * `MeterProviderBuilder` extension methods now support `OtlpExporterOptions` bound to `IConfiguration` when using OpenTelemetry.Extensions.Hosting ([#2413](https://github.com/open-telemetry/opentelemetry-dotnet/pull/2413)) +* Extended `OtlpExporterOptions` by `Protocol` property. The property can be + overridden by `OTEL_EXPORTER_OTLP_PROTOCOL` environmental variable (grpc or http/protobuf). + Implemented OTLP over HTTP binary protobuf trace exporter. + ([#2292](https://github.com/open-telemetry/opentelemetry-dotnet/issues/2292)) ## 1.2.0-alpha4 @@ -18,7 +22,7 @@ Released 2021-Sep-13 `BatchExportActivityProcessorOptions` which supports field value overriding using `OTEL_BSP_SCHEDULE_DELAY`, `OTEL_BSP_EXPORT_TIMEOUT`, `OTEL_BSP_MAX_QUEUE_SIZE`, `OTEL_BSP_MAX_EXPORT_BATCH_SIZE` - envionmental variables as defined in the + environmental variables as defined in the [specification](https://github.com/open-telemetry/opentelemetry-specification/blob/v1.5.0/specification/sdk-environment-variables.md#batch-span-processor). ([#2219](https://github.com/open-telemetry/opentelemetry-dotnet/pull/2219)) @@ -28,7 +32,7 @@ Released 2021-Aug-24 * The `OtlpExporterOptions` defaults can be overridden using `OTEL_EXPORTER_OTLP_ENDPOINT`, `OTEL_EXPORTER_OTLP_HEADERS` and `OTEL_EXPORTER_OTLP_TIMEOUT` - envionmental variables as defined in the + environmental variables as defined in the [specification](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/protocol/exporter.md). ([#2188](https://github.com/open-telemetry/opentelemetry-dotnet/pull/2188)) diff --git a/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/BaseOtlpExporter.cs b/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/Implementation/ExportClient/BaseOtlpGrpcExportClient.cs similarity index 56% rename from src/OpenTelemetry.Exporter.OpenTelemetryProtocol/BaseOtlpExporter.cs rename to src/OpenTelemetry.Exporter.OpenTelemetryProtocol/Implementation/ExportClient/BaseOtlpGrpcExportClient.cs index aad3503b2a2..c3dc00c19c5 100644 --- a/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/BaseOtlpExporter.cs +++ b/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/Implementation/ExportClient/BaseOtlpGrpcExportClient.cs @@ -1,4 +1,4 @@ -// +// // Copyright The OpenTelemetry Authors // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,54 +15,46 @@ // using System; +using System.Threading; using System.Threading.Tasks; using Grpc.Core; -using OpenTelemetry.Exporter.OpenTelemetryProtocol; -#if NETSTANDARD2_1 +#if NETSTANDARD2_1 || NET5_0_OR_GREATER using Grpc.Net.Client; #endif -using OpenTelemetry.Exporter.OpenTelemetryProtocol.Implementation; -using OtlpResource = Opentelemetry.Proto.Resource.V1; -namespace OpenTelemetry.Exporter +namespace OpenTelemetry.Exporter.OpenTelemetryProtocol.Implementation.ExportClient { - /// - /// Implements exporter that exports telemetry objects over OTLP/gRPC. - /// - /// The type of telemetry object to be exported. - public abstract class BaseOtlpExporter : BaseExporter - where T : class + /// Base class for sending OTLP export request over gRPC. + /// Type of export request. + internal abstract class BaseOtlpGrpcExportClient : IExportClient { - private OtlpResource.Resource processResource; - - /// - /// Initializes a new instance of the class. - /// - /// The for configuring the exporter. - protected BaseOtlpExporter(OtlpExporterOptions options) + protected BaseOtlpGrpcExportClient(OtlpExporterOptions options) { this.Options = options ?? throw new ArgumentNullException(nameof(options)); - this.Headers = options.GetMetadataFromHeaders(); - if (this.Options.TimeoutMilliseconds <= 0) + + if (options.TimeoutMilliseconds <= 0) { - throw new ArgumentException("Timeout value provided is not a positive number.", nameof(this.Options.TimeoutMilliseconds)); + throw new ArgumentException("Timeout value provided is not a positive number.", nameof(options.TimeoutMilliseconds)); } + + this.Headers = options.GetMetadataFromHeaders(); } - internal OtlpResource.Resource ProcessResource => this.processResource ??= this.ParentProvider.GetResource().ToOtlpResource(); + internal OtlpExporterOptions Options { get; } -#if NETSTANDARD2_1 +#if NETSTANDARD2_1 || NET5_0_OR_GREATER internal GrpcChannel Channel { get; set; } #else internal Channel Channel { get; set; } #endif - internal OtlpExporterOptions Options { get; } - internal Metadata Headers { get; } /// - protected override bool OnShutdown(int timeoutMilliseconds) + public abstract bool SendExportRequest(TRequest request, CancellationToken cancellationToken = default); + + /// + public virtual bool Shutdown(int timeoutMilliseconds) { if (this.Channel == null) { diff --git a/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/Implementation/ExportClient/BaseOtlpHttpExportClient.cs b/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/Implementation/ExportClient/BaseOtlpHttpExportClient.cs new file mode 100644 index 00000000000..3e5b27828fd --- /dev/null +++ b/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/Implementation/ExportClient/BaseOtlpHttpExportClient.cs @@ -0,0 +1,87 @@ +// +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +using System; +using System.Collections.Generic; +using System.Net.Http; +using System.Threading; + +namespace OpenTelemetry.Exporter.OpenTelemetryProtocol.Implementation.ExportClient +{ + /// Base class for sending OTLP export request over HTTP. + /// Type of export request. + internal abstract class BaseOtlpHttpExportClient : IExportClient + { + protected BaseOtlpHttpExportClient(OtlpExporterOptions options, HttpClient httpClient = null) + { + this.Options = options ?? throw new ArgumentNullException(nameof(options)); + + if (this.Options.TimeoutMilliseconds <= 0) + { + throw new ArgumentException("Timeout value provided is not a positive number.", nameof(this.Options.TimeoutMilliseconds)); + } + + this.Headers = options.GetHeaders>((d, k, v) => d.Add(k, v)); + + this.HttpClient = httpClient ?? new HttpClient { Timeout = TimeSpan.FromMilliseconds(this.Options.TimeoutMilliseconds) }; + } + + internal OtlpExporterOptions Options { get; } + + internal HttpClient HttpClient { get; } + + internal IReadOnlyDictionary Headers { get; } + + /// + public bool SendExportRequest(TRequest request, CancellationToken cancellationToken = default) + { + try + { + using var httpRequest = this.CreateHttpRequest(request); + + using var httpResponse = this.SendHttpRequest(httpRequest, cancellationToken); + + httpResponse?.EnsureSuccessStatusCode(); + } + catch (HttpRequestException ex) + { + OpenTelemetryProtocolExporterEventSource.Log.FailedToReachCollector(ex); + + return false; + } + + return true; + } + + /// + public bool Shutdown(int timeoutMilliseconds) + { + this.HttpClient.CancelPendingRequests(); + return true; + } + + protected abstract HttpRequestMessage CreateHttpRequest(TRequest request); + + protected HttpResponseMessage SendHttpRequest(HttpRequestMessage request, CancellationToken cancellationToken) + { +#if NET5_0_OR_GREATER + return this.HttpClient.Send(request, cancellationToken); +#else + return this.HttpClient.SendAsync(request, cancellationToken).GetAwaiter().GetResult(); +#endif + } + } +} diff --git a/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/Implementation/ExportClient/IExportClient.cs b/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/Implementation/ExportClient/IExportClient.cs new file mode 100644 index 00000000000..61d85bacdad --- /dev/null +++ b/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/Implementation/ExportClient/IExportClient.cs @@ -0,0 +1,45 @@ +// +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +using System.Threading; + +namespace OpenTelemetry.Exporter.OpenTelemetryProtocol.Implementation.ExportClient +{ + /// Export client interface. + /// Type of export request. + internal interface IExportClient + { + /// + /// Method for sending export request to the server. + /// + /// The request to send to the server. + /// An optional token for canceling the call. + /// True if the request has been sent successfully, otherwise false. + bool SendExportRequest(TRequest request, CancellationToken cancellationToken = default); + + /// + /// Method for shutting down the export client. + /// + /// + /// The number of milliseconds to wait, or Timeout.Infinite to + /// wait indefinitely. + /// + /// + /// Returns true if shutdown succeeded; otherwise, false. + /// + bool Shutdown(int timeoutMilliseconds); + } +} diff --git a/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/Implementation/ExportClient/OtlpGrpcMetricsExportClient.cs b/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/Implementation/ExportClient/OtlpGrpcMetricsExportClient.cs new file mode 100644 index 00000000000..6d904a73a8f --- /dev/null +++ b/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/Implementation/ExportClient/OtlpGrpcMetricsExportClient.cs @@ -0,0 +1,62 @@ +// +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +using System; +using System.Threading; +using Grpc.Core; +using OtlpCollector = Opentelemetry.Proto.Collector.Metrics.V1; + +namespace OpenTelemetry.Exporter.OpenTelemetryProtocol.Implementation.ExportClient +{ + /// Class for sending OTLP metrics export request over gRPC. + internal sealed class OtlpGrpcMetricsExportClient : BaseOtlpGrpcExportClient + { + private readonly OtlpCollector.MetricsService.IMetricsServiceClient metricsClient; + + public OtlpGrpcMetricsExportClient(OtlpExporterOptions options, OtlpCollector.MetricsService.IMetricsServiceClient metricsServiceClient = null) + : base(options) + { + if (metricsServiceClient != null) + { + this.metricsClient = metricsServiceClient; + } + else + { + this.Channel = options.CreateChannel(); + this.metricsClient = new OtlpCollector.MetricsService.MetricsServiceClient(this.Channel); + } + } + + /// + public override bool SendExportRequest(OtlpCollector.ExportMetricsServiceRequest request, CancellationToken cancellationToken = default) + { + var deadline = DateTime.UtcNow.AddMilliseconds(this.Options.TimeoutMilliseconds); + + try + { + this.metricsClient.Export(request, headers: this.Headers, deadline: deadline); + } + catch (RpcException ex) + { + OpenTelemetryProtocolExporterEventSource.Log.FailedToReachCollector(ex); + + return false; + } + + return true; + } + } +} diff --git a/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/Implementation/ExportClient/OtlpGrpcTraceExportClient.cs b/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/Implementation/ExportClient/OtlpGrpcTraceExportClient.cs new file mode 100644 index 00000000000..92af4c854a8 --- /dev/null +++ b/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/Implementation/ExportClient/OtlpGrpcTraceExportClient.cs @@ -0,0 +1,62 @@ +// +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +using System; +using System.Threading; +using Grpc.Core; +using OtlpCollector = Opentelemetry.Proto.Collector.Trace.V1; + +namespace OpenTelemetry.Exporter.OpenTelemetryProtocol.Implementation.ExportClient +{ + /// Class for sending OTLP trace export request over gRPC. + internal sealed class OtlpGrpcTraceExportClient : BaseOtlpGrpcExportClient + { + private readonly OtlpCollector.TraceService.ITraceServiceClient traceClient; + + public OtlpGrpcTraceExportClient(OtlpExporterOptions options, OtlpCollector.TraceService.ITraceServiceClient traceServiceClient = null) + : base(options) + { + if (traceServiceClient != null) + { + this.traceClient = traceServiceClient; + } + else + { + this.Channel = options.CreateChannel(); + this.traceClient = new OtlpCollector.TraceService.TraceServiceClient(this.Channel); + } + } + + /// + public override bool SendExportRequest(OtlpCollector.ExportTraceServiceRequest request, CancellationToken cancellationToken = default) + { + var deadline = DateTime.UtcNow.AddMilliseconds(this.Options.TimeoutMilliseconds); + + try + { + this.traceClient.Export(request, headers: this.Headers, deadline: deadline); + } + catch (RpcException ex) + { + OpenTelemetryProtocolExporterEventSource.Log.FailedToReachCollector(ex); + + return false; + } + + return true; + } + } +} diff --git a/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/Implementation/ExportClient/OtlpHttpTraceExportClient.cs b/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/Implementation/ExportClient/OtlpHttpTraceExportClient.cs new file mode 100644 index 00000000000..9bb7e04fb27 --- /dev/null +++ b/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/Implementation/ExportClient/OtlpHttpTraceExportClient.cs @@ -0,0 +1,60 @@ +// +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +using System; +using System.IO; +using System.Net.Http; +using System.Net.Http.Headers; +using Google.Protobuf; +using OtlpCollector = Opentelemetry.Proto.Collector.Trace.V1; + +namespace OpenTelemetry.Exporter.OpenTelemetryProtocol.Implementation.ExportClient +{ + /// Class for sending OTLP trace export request over HTTP. + internal sealed class OtlpHttpTraceExportClient : BaseOtlpHttpExportClient + { + internal const string MediaContentType = "application/x-protobuf"; + private readonly Uri exportTracesUri; + + public OtlpHttpTraceExportClient(OtlpExporterOptions options, HttpClient httpClient = null) + : base(options, httpClient) + { + this.exportTracesUri = this.Options.Endpoint.AppendPathIfNotPresent(OtlpExporterOptions.TracesExportPath); + } + + protected override HttpRequestMessage CreateHttpRequest(OtlpCollector.ExportTraceServiceRequest exportRequest) + { + var request = new HttpRequestMessage(HttpMethod.Post, this.exportTracesUri); + foreach (var header in this.Headers) + { + request.Headers.Add(header.Key, header.Value); + } + + var content = Array.Empty(); + using (var stream = new MemoryStream()) + { + exportRequest.WriteTo(stream); + content = stream.ToArray(); + } + + var binaryContent = new ByteArrayContent(content); + binaryContent.Headers.ContentType = new MediaTypeHeaderValue(MediaContentType); + request.Content = binaryContent; + + return request; + } + } +} diff --git a/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/Implementation/OpenTelemetryProtocolExporterEventSource.cs b/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/Implementation/OpenTelemetryProtocolExporterEventSource.cs index 1824c82fb69..bfff7ad0443 100644 --- a/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/Implementation/OpenTelemetryProtocolExporterEventSource.cs +++ b/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/Implementation/OpenTelemetryProtocolExporterEventSource.cs @@ -103,5 +103,11 @@ public void MissingPermissionsToReadEnvironmentVariable(string exception) { this.WriteEvent(7, exception); } + + [Event(8, Message = "Unsupported value for protocol '{0}' is configured, default protocol 'grpc' will be used.", Level = EventLevel.Warning)] + public void UnsupportedProtocol(string protocol) + { + this.WriteEvent(8, protocol); + } } } diff --git a/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/OpenTelemetry.Exporter.OpenTelemetryProtocol.csproj b/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/OpenTelemetry.Exporter.OpenTelemetryProtocol.csproj index ecee7ccec98..fa9ece21ad7 100644 --- a/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/OpenTelemetry.Exporter.OpenTelemetryProtocol.csproj +++ b/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/OpenTelemetry.Exporter.OpenTelemetryProtocol.csproj @@ -1,9 +1,11 @@  - netstandard2.0;netstandard2.1;net461 + netstandard2.0;netstandard2.1;net461;net5.0 OpenTelemetry protocol exporter for OpenTelemetry .NET $(PackageTags);OTLP core- + + false + + false + + + + + + + diff --git a/src/OpenTelemetry.Exporter.OpenTelemetryProtocol.Logs/OtlpLogExporterHelperExtensions.cs b/src/OpenTelemetry.Exporter.OpenTelemetryProtocol.Logs/OtlpLogExporterHelperExtensions.cs new file mode 100644 index 00000000000..9df4911f76f --- /dev/null +++ b/src/OpenTelemetry.Exporter.OpenTelemetryProtocol.Logs/OtlpLogExporterHelperExtensions.cs @@ -0,0 +1,63 @@ +// +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +using System; +using OpenTelemetry.Exporter; + +namespace OpenTelemetry.Logs +{ + /// + /// Extension methods to simplify registering of the OpenTelemetry Protocol (OTLP) exporter. + /// + public static class OtlpLogExporterHelperExtensions + { + /// + /// Adds OTLP Exporter as a configuration to the OpenTelemetry ILoggingBuilder. + /// + /// options to use. + /// Exporter configuration options. + /// The instance of to chain the calls. + public static OpenTelemetryLoggerOptions AddOtlpExporter(this OpenTelemetryLoggerOptions loggerOptions, Action configure = null) + { + if (loggerOptions == null) + { + throw new ArgumentNullException(nameof(loggerOptions)); + } + + return AddOtlpExporter(loggerOptions, new OtlpExporterOptions(), configure); + } + + private static OpenTelemetryLoggerOptions AddOtlpExporter(OpenTelemetryLoggerOptions loggerOptions, OtlpExporterOptions exporterOptions, Action configure = null) + { + configure?.Invoke(exporterOptions); + var otlpExporter = new OtlpLogExporter(exporterOptions); + + if (exporterOptions.ExportProcessorType == ExportProcessorType.Simple) + { + return loggerOptions.AddProcessor(new SimpleLogRecordExportProcessor(otlpExporter)); + } + else + { + return loggerOptions.AddProcessor(new BatchLogRecordExportProcessor( + otlpExporter, + exporterOptions.BatchExportProcessorOptions.MaxQueueSize, + exporterOptions.BatchExportProcessorOptions.ScheduledDelayMilliseconds, + exporterOptions.BatchExportProcessorOptions.ExporterTimeoutMilliseconds, + exporterOptions.BatchExportProcessorOptions.MaxExportBatchSize)); + } + } + } +} diff --git a/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/AssemblyInfo.cs b/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/AssemblyInfo.cs index 8e90458d1fc..46798ca4e77 100644 --- a/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/AssemblyInfo.cs +++ b/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/AssemblyInfo.cs @@ -19,6 +19,7 @@ #if SIGNED [assembly: InternalsVisibleTo("OpenTelemetry.Exporter.OpenTelemetryProtocol.Tests, PublicKey=002400000480000094000000060200000024000052534131000400000100010051c1562a090fb0c9f391012a32198b5e5d9a60e9b80fa2d7b434c9e5ccb7259bd606e66f9660676afc6692b8cdc6793d190904551d2103b7b22fa636dcbb8208839785ba402ea08fc00c8f1500ccef28bbf599aa64ffb1e1d5dc1bf3420a3777badfe697856e9d52070a50c3ea5821c80bef17ca3acffa28f89dd413f096f898")] [assembly: InternalsVisibleTo("Benchmarks, PublicKey=002400000480000094000000060200000024000052534131000400000100010051c1562a090fb0c9f391012a32198b5e5d9a60e9b80fa2d7b434c9e5ccb7259bd606e66f9660676afc6692b8cdc6793d190904551d2103b7b22fa636dcbb8208839785ba402ea08fc00c8f1500ccef28bbf599aa64ffb1e1d5dc1bf3420a3777badfe697856e9d52070a50c3ea5821c80bef17ca3acffa28f89dd413f096f898")] +[assembly: InternalsVisibleTo("OpenTelemetry.Exporter.OpenTelemetryProtocol.Logs, PublicKey=002400000480000094000000060200000024000052534131000400000100010051c1562a090fb0c9f391012a32198b5e5d9a60e9b80fa2d7b434c9e5ccb7259bd606e66f9660676afc6692b8cdc6793d190904551d2103b7b22fa636dcbb8208839785ba402ea08fc00c8f1500ccef28bbf599aa64ffb1e1d5dc1bf3420a3777badfe697856e9d52070a50c3ea5821c80bef17ca3acffa28f89dd413f096f898")] // Used by Moq. [assembly: InternalsVisibleTo("DynamicProxyGenAssembly2, PublicKey=0024000004800000940000000602000000240000525341310004000001000100c547cac37abd99c8db225ef2f6c8a3602f3b3606cc9891605d02baa56104f4cfc0734aa39b93bf7852f7d9266654753cc297e7d2edfe0bac1cdcf9f717241550e0a7b191195b7667bb4f64bcb8e2121380fd1d9d46ad2d92d2d15605093924cceaf74c4861eff62abf69b9291ed0a340e113be11e6a7d3113e92484cf7045cc7")] diff --git a/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/Implementation/ExportClient/OtlpGrpcLogExportClient.cs b/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/Implementation/ExportClient/OtlpGrpcLogExportClient.cs new file mode 100644 index 00000000000..48c93d9eae8 --- /dev/null +++ b/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/Implementation/ExportClient/OtlpGrpcLogExportClient.cs @@ -0,0 +1,62 @@ +// +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +using System; +using System.Threading; +using Grpc.Core; +using OtlpCollector = Opentelemetry.Proto.Collector.Logs.V1; + +namespace OpenTelemetry.Exporter.OpenTelemetryProtocol.Implementation.ExportClient +{ + /// Class for sending OTLP metrics export request over gRPC. + internal sealed class OtlpGrpcLogExportClient : BaseOtlpGrpcExportClient + { + private readonly OtlpCollector.LogsService.ILogsServiceClient logsClient; + + public OtlpGrpcLogExportClient(OtlpExporterOptions options, OtlpCollector.LogsService.ILogsServiceClient logsServiceClient = null) + : base(options) + { + if (logsServiceClient != null) + { + this.logsClient = logsServiceClient; + } + else + { + this.Channel = options.CreateChannel(); + this.logsClient = new OtlpCollector.LogsService.LogsServiceClient(this.Channel); + } + } + + /// + public override bool SendExportRequest(OtlpCollector.ExportLogsServiceRequest request, CancellationToken cancellationToken = default) + { + var deadline = DateTime.UtcNow.AddMilliseconds(this.Options.TimeoutMilliseconds); + + try + { + this.logsClient.Export(request, headers: this.Headers, deadline: deadline); + } + catch (RpcException ex) + { + OpenTelemetryProtocolExporterEventSource.Log.FailedToReachCollector(ex); + + return false; + } + + return true; + } + } +} diff --git a/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/Implementation/LogRecordExtensions.cs b/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/Implementation/LogRecordExtensions.cs new file mode 100644 index 00000000000..69cb934a994 --- /dev/null +++ b/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/Implementation/LogRecordExtensions.cs @@ -0,0 +1,121 @@ +// +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +using System.Collections.Generic; +using System.Runtime.CompilerServices; +using Google.Protobuf; +using Google.Protobuf.Collections; +using OpenTelemetry.Internal; +using OpenTelemetry.Logs; +using OpenTelemetry.Trace; +using OtlpCollector = Opentelemetry.Proto.Collector.Logs.V1; +using OtlpCommon = Opentelemetry.Proto.Common.V1; +using OtlpLogs = Opentelemetry.Proto.Logs.V1; +using OtlpResource = Opentelemetry.Proto.Resource.V1; + +namespace OpenTelemetry.Exporter.OpenTelemetryProtocol.Implementation +{ + internal static class LogRecordExtensions + { + internal static void AddBatch( + this OtlpCollector.ExportLogsServiceRequest request, + OtlpResource.Resource processResource, + in Batch logRecordBatch) + { + Dictionary logRecordsByLibrary = new Dictionary(); + OtlpLogs.ResourceLogs resourceLogs = new OtlpLogs.ResourceLogs + { + Resource = processResource, + }; + request.ResourceLogs.Add(resourceLogs); + + var instrumentationLibraryLogs = new OtlpLogs.InstrumentationLibraryLogs(); + resourceLogs.InstrumentationLibraryLogs.Add(instrumentationLibraryLogs); + + foreach (var item in logRecordBatch) + { + var logRecord = item.ToOtlpLog(); + if (logRecord == null) + { + OpenTelemetryProtocolExporterEventSource.Log.CouldNotTranslateLogRecord( + nameof(LogRecordExtensions), + nameof(AddBatch)); + continue; + } + + instrumentationLibraryLogs.Logs.Add(logRecord); + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static OtlpLogs.LogRecord ToOtlpLog(this LogRecord logRecord) + { + var otlpLogRecord = new OtlpLogs.LogRecord + { + TimeUnixNano = (ulong)logRecord.Timestamp.ToUnixTimeNanoseconds(), + Name = logRecord.CategoryName, + + // TODO: Devise mapping of LogLevel to SeverityNumber + // See: https://github.com/open-telemetry/opentelemetry-proto/blob/bacfe08d84e21fb2a779e302d12e8dfeb67e7b86/opentelemetry/proto/logs/v1/logs.proto#L100-L102 + SeverityText = logRecord.LogLevel.ToString(), + }; + + if (logRecord.FormattedMessage != null) + { + otlpLogRecord.Body = new OtlpCommon.AnyValue { StringValue = logRecord.FormattedMessage }; + } + + if (logRecord.EventId != 0) + { + otlpLogRecord.Attributes.AddAttribute(nameof(logRecord.EventId), logRecord.EventId.ToString()); + } + + if (logRecord.Exception != null) + { + otlpLogRecord.Attributes.AddAttribute(SemanticConventions.AttributeExceptionType, logRecord.Exception.GetType().Name); + otlpLogRecord.Attributes.AddAttribute(SemanticConventions.AttributeExceptionMessage, logRecord.Exception.Message); + otlpLogRecord.Attributes.AddAttribute(SemanticConventions.AttributeExceptionStacktrace, logRecord.Exception.ToInvariantString()); + } + + if (logRecord.TraceId != default && logRecord.SpanId != default) + { + byte[] traceIdBytes = new byte[16]; + byte[] spanIdBytes = new byte[8]; + + logRecord.TraceId.CopyTo(traceIdBytes); + logRecord.SpanId.CopyTo(spanIdBytes); + + otlpLogRecord.TraceId = UnsafeByteOperations.UnsafeWrap(traceIdBytes); + otlpLogRecord.SpanId = UnsafeByteOperations.UnsafeWrap(spanIdBytes); + otlpLogRecord.Flags = (uint)logRecord.TraceFlags; + } + + // TODO: Add additional attributes from scope and state + // Might make sense to take an approach similar to https://github.com/open-telemetry/opentelemetry-dotnet-contrib/blob/897b734aa5ea9992538f04f6ea6871fe211fa903/src/OpenTelemetry.Contrib.Preview/Internal/DefaultLogStateConverter.cs + + return otlpLogRecord; + } + + private static void AddAttribute(this RepeatedField repeatedField, string key, string value) + { + repeatedField.Add(new OtlpCommon.KeyValue + { + Key = key, + Value = new OtlpCommon.AnyValue { StringValue = value }, + }); + } + } +} diff --git a/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/Implementation/LogsService.cs b/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/Implementation/LogsService.cs new file mode 100644 index 00000000000..7355c0153a4 --- /dev/null +++ b/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/Implementation/LogsService.cs @@ -0,0 +1,50 @@ +// +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +using System; +using System.Threading; +using Grpc.Core; + +namespace Opentelemetry.Proto.Collector.Logs.V1 +{ + /// + /// LogService extensions. + /// + internal static partial class LogsService + { + /// Interface for LogService. + public interface ILogsServiceClient + { + /// + /// For performance reasons, it is recommended to keep this RPC + /// alive for the entire life of the application. + /// + /// The request to send to the server. + /// The initial metadata to send with the call. This parameter is optional. + /// An optional deadline for the call. The call will be cancelled if deadline is hit. + /// An optional token for canceling the call. + /// The response received from the server. + ExportLogsServiceResponse Export(ExportLogsServiceRequest request, Metadata headers = null, DateTime? deadline = null, CancellationToken cancellationToken = default); + } + + /// + /// LogsServiceClient extensions. + /// + public partial class LogsServiceClient : ILogsServiceClient + { + } + } +} diff --git a/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/Implementation/OpenTelemetryProtocolExporterEventSource.cs b/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/Implementation/OpenTelemetryProtocolExporterEventSource.cs index bfff7ad0443..8aecdf51f07 100644 --- a/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/Implementation/OpenTelemetryProtocolExporterEventSource.cs +++ b/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/Implementation/OpenTelemetryProtocolExporterEventSource.cs @@ -109,5 +109,11 @@ public void UnsupportedProtocol(string protocol) { this.WriteEvent(8, protocol); } + + [Event(9, Message = "Could not translate LogRecord from class '{0}' and method '{1}', log will not be exported.", Level = EventLevel.Informational)] + public void CouldNotTranslateLogRecord(string className, string methodName) + { + this.WriteEvent(9, className, methodName); + } } } diff --git a/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/OtlpLogExporter.cs b/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/OtlpLogExporter.cs new file mode 100644 index 00000000000..c033436b392 --- /dev/null +++ b/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/OtlpLogExporter.cs @@ -0,0 +1,97 @@ +// +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +using System; +using OpenTelemetry.Exporter.OpenTelemetryProtocol.Implementation; +using OpenTelemetry.Exporter.OpenTelemetryProtocol.Implementation.ExportClient; +using OpenTelemetry.Logs; +using OtlpCollector = Opentelemetry.Proto.Collector.Logs.V1; +using OtlpResource = Opentelemetry.Proto.Resource.V1; + +namespace OpenTelemetry.Exporter +{ + /// + /// Exporter consuming and exporting the data using + /// the OpenTelemetry protocol (OTLP). + /// + internal class OtlpLogExporter : BaseExporter + { + private readonly IExportClient exportClient; + + private OtlpResource.Resource processResource; + + /// + /// Initializes a new instance of the class. + /// + /// Configuration options for the exporter. + public OtlpLogExporter(OtlpExporterOptions options) + : this(options, null) + { + } + + /// + /// Initializes a new instance of the class. + /// + /// Configuration options for the exporter. + /// Client used for sending export request. + internal OtlpLogExporter(OtlpExporterOptions options, IExportClient exportClient = null) + { + if (exportClient != null) + { + this.exportClient = exportClient; + } + else + { + // TODO: this instantiation should be aligned with the protocol option (grpc or http/protobuf) when OtlpHttpMetricsExportClient will be implemented. + this.exportClient = new OtlpGrpcLogExportClient(options); + } + } + + internal OtlpResource.Resource ProcessResource => this.processResource ??= this.ParentProvider.GetResource().ToOtlpResource(); + + /// + public override ExportResult Export(in Batch logRecordBatch) + { + // Prevents the exporter's gRPC and HTTP operations from being instrumented. + using var scope = SuppressInstrumentationScope.Begin(); + + var request = new OtlpCollector.ExportLogsServiceRequest(); + + request.AddBatch(this.ProcessResource, logRecordBatch); + + try + { + if (!this.exportClient.SendExportRequest(request)) + { + return ExportResult.Failure; + } + } + catch (Exception ex) + { + OpenTelemetryProtocolExporterEventSource.Log.ExportMethodException(ex); + return ExportResult.Failure; + } + + return ExportResult.Success; + } + + /// + protected override bool OnShutdown(int timeoutMilliseconds) + { + return this.exportClient?.Shutdown(timeoutMilliseconds) ?? true; + } + } +} diff --git a/src/OpenTelemetry/Metrics/BaseExportingMetricReader.cs b/src/OpenTelemetry/Metrics/BaseExportingMetricReader.cs index aae8da125b6..2fe914f9fa9 100644 --- a/src/OpenTelemetry/Metrics/BaseExportingMetricReader.cs +++ b/src/OpenTelemetry/Metrics/BaseExportingMetricReader.cs @@ -78,6 +78,7 @@ internal override void SetParentProvider(BaseProvider parentProvider) /// protected override bool ProcessMetrics(Batch metrics, int timeoutMilliseconds) { + // TODO: Do we need to consider timeout here? return this.exporter.Export(metrics) == ExportResult.Success; } diff --git a/src/OpenTelemetry/Metrics/PeriodicExportingMetricReader.cs b/src/OpenTelemetry/Metrics/PeriodicExportingMetricReader.cs index faad540e12e..54b5a6dcc4c 100644 --- a/src/OpenTelemetry/Metrics/PeriodicExportingMetricReader.cs +++ b/src/OpenTelemetry/Metrics/PeriodicExportingMetricReader.cs @@ -47,6 +47,7 @@ public PeriodicExportingMetricReader( { while (!this.token.IsCancellationRequested) { + // TODO: Should pass CancellationToken Task.Delay(exportIntervalMilliseconds).Wait(); this.Collect(); } From 1a9ec3d92d13808e6e8d59aa90c4730f68f00c46 Mon Sep 17 00:00:00 2001 From: Mikel Blanchard Date: Thu, 7 Oct 2021 13:31:52 -0700 Subject: [PATCH 29/39] Added integration test for PrometheusExporterMetricsHttpServer. (#2465) Co-authored-by: Cijo Thomas --- .../PrometheusExporter.cs | 3 +- ...rometheusExporterMetricsHttpServerTests.cs | 111 ++++++++++++++++++ .../PrometheusExporterMiddlewareTests.cs | 2 +- 3 files changed, 114 insertions(+), 2 deletions(-) create mode 100644 test/OpenTelemetry.Exporter.Prometheus.Tests/PrometheusExporterMetricsHttpServerTests.cs diff --git a/src/OpenTelemetry.Exporter.Prometheus/PrometheusExporter.cs b/src/OpenTelemetry.Exporter.Prometheus/PrometheusExporter.cs index 66bd484116f..6d8e3ad25d0 100644 --- a/src/OpenTelemetry.Exporter.Prometheus/PrometheusExporter.cs +++ b/src/OpenTelemetry.Exporter.Prometheus/PrometheusExporter.cs @@ -28,6 +28,7 @@ namespace OpenTelemetry.Exporter [ExportModes(ExportModes.Pull)] public class PrometheusExporter : BaseExporter, IPullMetricExporter { + internal const string HttpListenerStartFailureExceptionMessage = "PrometheusExporter http listener could not be started."; internal readonly PrometheusExporterOptions Options; internal Batch Metrics; private readonly SemaphoreSlim semaphore = new SemaphoreSlim(1, 1); @@ -52,7 +53,7 @@ public PrometheusExporter(PrometheusExporterOptions options) } catch (Exception ex) { - throw new InvalidOperationException("PrometheusExporter http listener could not be started.", ex); + throw new InvalidOperationException(HttpListenerStartFailureExceptionMessage, ex); } } } diff --git a/test/OpenTelemetry.Exporter.Prometheus.Tests/PrometheusExporterMetricsHttpServerTests.cs b/test/OpenTelemetry.Exporter.Prometheus.Tests/PrometheusExporterMetricsHttpServerTests.cs new file mode 100644 index 00000000000..7fdf5dca4cc --- /dev/null +++ b/test/OpenTelemetry.Exporter.Prometheus.Tests/PrometheusExporterMetricsHttpServerTests.cs @@ -0,0 +1,111 @@ +// +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +using System; +using System.Collections.Generic; +using System.Diagnostics.Metrics; +using System.Net; +using System.Net.Http; +using System.Threading.Tasks; +using OpenTelemetry.Metrics; +using Xunit; + +namespace OpenTelemetry.Exporter.Prometheus.Tests +{ + public class PrometheusExporterMetricsHttpServerTests + { + private const string MeterName = "PrometheusExporterMetricsHttpServerTests.Meter"; + + [Fact] + public async Task PrometheusExporterMetricsHttpServerIntegration() + { + Random random = new Random(); + int port = 0; + int retryCount = 5; + MeterProvider provider; + string address = null; + + while (true) + { + try + { + port = random.Next(2000, 5000); + provider = Sdk.CreateMeterProviderBuilder() + .AddMeter(MeterName) + .AddPrometheusExporter(o => + { + o.GetUtcNowDateTimeOffset = () => new DateTimeOffset(2021, 9, 30, 22, 30, 0, TimeSpan.Zero); +#if NET461 + bool expectedDefaultState = true; +#else + bool expectedDefaultState = false; +#endif + if (o.StartHttpListener != expectedDefaultState) + { + throw new InvalidOperationException("StartHttpListener value is unexpected."); + } + + if (!o.StartHttpListener) + { + o.StartHttpListener = true; + } + + address = $"http://localhost:{port}/"; + o.HttpListenerPrefixes = new string[] { address }; + }) + .Build(); + break; + } + catch (Exception ex) + { + if (ex.Message != PrometheusExporter.HttpListenerStartFailureExceptionMessage) + { + throw; + } + + if (retryCount-- <= 0) + { + throw new InvalidOperationException("HttpListener could not be started."); + } + } + } + + var tags = new KeyValuePair[] + { + new KeyValuePair("key1", "value1"), + new KeyValuePair("key2", "value2"), + }; + + using var meter = new Meter(MeterName, "0.0.1"); + + var counter = meter.CreateCounter("counter_double"); + counter.Add(100.18D, tags); + counter.Add(0.99D, tags); + + using HttpClient client = new HttpClient(); + + using var response = await client.GetAsync($"{address}metrics").ConfigureAwait(false); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + + string content = await response.Content.ReadAsStringAsync().ConfigureAwait(false); + + Assert.Equal( + $"# TYPE counter_double counter\ncounter_double{{key1=\"value1\",key2=\"value2\"}} 101.17 1633041000000\n", + content); + } + } +} diff --git a/test/OpenTelemetry.Exporter.Prometheus.Tests/PrometheusExporterMiddlewareTests.cs b/test/OpenTelemetry.Exporter.Prometheus.Tests/PrometheusExporterMiddlewareTests.cs index 3c90c3ec813..bcc3c9645c1 100644 --- a/test/OpenTelemetry.Exporter.Prometheus.Tests/PrometheusExporterMiddlewareTests.cs +++ b/test/OpenTelemetry.Exporter.Prometheus.Tests/PrometheusExporterMiddlewareTests.cs @@ -55,7 +55,7 @@ public async Task PrometheusExporterMiddlewareIntegration() counter.Add(100.18D, tags); counter.Add(0.99D, tags); - using var response = await host.GetTestClient().GetAsync("/metrics"); + using var response = await host.GetTestClient().GetAsync("/metrics").ConfigureAwait(false); Assert.Equal(HttpStatusCode.OK, response.StatusCode); From c3cef632104cf454a45f0502d116fd445140c43b Mon Sep 17 00:00:00 2001 From: Cijo Thomas Date: Thu, 7 Oct 2021 18:03:09 -0700 Subject: [PATCH 30/39] Error logging for Metrics (#2458) * Error logging for Metrics * rename Fix to action * typo --- .../Internal/OpenTelemetrySdkEventSource.cs | 6 ++ src/OpenTelemetry/Metrics/MeterProviderSdk.cs | 56 +++++++++++-------- 2 files changed, 38 insertions(+), 24 deletions(-) diff --git a/src/OpenTelemetry/Internal/OpenTelemetrySdkEventSource.cs b/src/OpenTelemetry/Internal/OpenTelemetrySdkEventSource.cs index 67042d9d9e4..402de38c0ee 100644 --- a/src/OpenTelemetry/Internal/OpenTelemetrySdkEventSource.cs +++ b/src/OpenTelemetry/Internal/OpenTelemetrySdkEventSource.cs @@ -350,6 +350,12 @@ public void ExistsDroppedExportProcessorItems(string exportProcessorName, string this.WriteEvent(32, exportProcessorName, exporterName, droppedCount); } + [Event(33, Message = "Measurements from Instrument '{0}', Meter '{1}' will be ignored. Reason: '{1}'. Suggested action: '{2}'", Level = EventLevel.Warning)] + public void MetricInstrumentIgnored(string instrumentName, string meterName, string reason, string fix) + { + this.WriteEvent(33, instrumentName, meterName, reason, fix); + } + #if DEBUG public class OpenTelemetryEventListener : EventListener { diff --git a/src/OpenTelemetry/Metrics/MeterProviderSdk.cs b/src/OpenTelemetry/Metrics/MeterProviderSdk.cs index 771412bc428..8c12d1b62bb 100644 --- a/src/OpenTelemetry/Metrics/MeterProviderSdk.cs +++ b/src/OpenTelemetry/Metrics/MeterProviderSdk.cs @@ -197,36 +197,44 @@ internal MeterProviderSdk( { this.listener.InstrumentPublished = (instrument, listener) => { - if (meterSourcesToSubscribe.ContainsKey(instrument.Meter.Name)) + try { - var metricName = instrument.Name; - Metric metric = null; - lock (this.instrumentCreationLock) + if (meterSourcesToSubscribe.ContainsKey(instrument.Meter.Name)) { - if (this.metricStreamNames.ContainsKey(metricName)) + var metricName = instrument.Name; + Metric metric = null; + lock (this.instrumentCreationLock) { - // TODO: Log that instrument is ignored - // as the resulting Metric name is conflicting - // with existing name. - return; - } + if (this.metricStreamNames.ContainsKey(metricName)) + { + OpenTelemetrySdkEventSource.Log.MetricInstrumentIgnored(metricName, instrument.Meter.Name, "Metric name conflicting with existing name.", "Either change the name of the instrument or change name using View."); + return; + } - var index = ++this.metricIndex; - if (index >= MaxMetrics) - { - // TODO: Log that instrument is ignored - // as max number of Metrics have reached. - return; - } - else - { - metric = new Metric(instrument, temporality, metricName, instrument.Description); - this.metrics[index] = metric; - this.metricStreamNames.Add(metricName, true); + var index = ++this.metricIndex; + if (index >= MaxMetrics) + { + OpenTelemetrySdkEventSource.Log.MetricInstrumentIgnored(metricName, instrument.Meter.Name, "Maximum allowed Metrics for the provider exceeded.", "Use views to drop unused instruments. Or configure Provider to allow higher limit."); + return; + } + else + { + metric = new Metric(instrument, temporality, metricName, instrument.Description); + this.metrics[index] = metric; + this.metricStreamNames.Add(metricName, true); + } } - } - listener.EnableMeasurementEvents(instrument, metric); + listener.EnableMeasurementEvents(instrument, metric); + } + else + { + OpenTelemetrySdkEventSource.Log.MetricInstrumentIgnored(instrument.Name, instrument.Meter.Name, "Instrument belongs to a Meter not subscribed by the provider.", "Use AddMeter to add the Meter to the provider."); + } + } + catch (Exception) + { + OpenTelemetrySdkEventSource.Log.MetricInstrumentIgnored(instrument.Name, instrument.Meter.Name, "SDK internal error occurred.", "Contact SDK owners."); } }; From db8f0e712555716932e323f5fc7c301b17ec4c11 Mon Sep 17 00:00:00 2001 From: Utkarsh Umesan Pillai Date: Thu, 7 Oct 2021 20:02:19 -0700 Subject: [PATCH 31/39] Rename variable name to MetricPoints (#2468) --- src/OpenTelemetry/Metrics/AggregatorStore.cs | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/OpenTelemetry/Metrics/AggregatorStore.cs b/src/OpenTelemetry/Metrics/AggregatorStore.cs index 503cf510bf3..d1344a69fc7 100644 --- a/src/OpenTelemetry/Metrics/AggregatorStore.cs +++ b/src/OpenTelemetry/Metrics/AggregatorStore.cs @@ -36,7 +36,7 @@ internal sealed class AggregatorStore private readonly AggregationTemporality temporality; private readonly bool outputDelta; - private readonly MetricPoint[] metrics; + private readonly MetricPoint[] metricPoints; private readonly AggregationType aggType; private readonly double[] histogramBounds; private readonly UpdateLongDelegate updateLongCallback; @@ -52,7 +52,7 @@ internal AggregatorStore( double[] histogramBounds, string[] tagKeysInteresting = null) { - this.metrics = new MetricPoint[MaxMetricPoints]; + this.metricPoints = new MetricPoint[MaxMetricPoints]; this.aggType = aggType; this.temporality = temporality; this.outputDelta = temporality == AggregationTemporality.Delta ? true : false; @@ -98,7 +98,7 @@ internal void SnapShot() for (int i = 0; i <= indexSnapShot; i++) { - ref var metricPoint = ref this.metrics[i]; + ref var metricPoint = ref this.metricPoints[i]; if (metricPoint.StartTime == default) { continue; @@ -122,7 +122,7 @@ internal void SnapShot() internal BatchMetricPoint GetMetricPoints() { var indexSnapShot = Math.Min(this.metricPointIndex, MaxMetricPoints - 1); - return new BatchMetricPoint(this.metrics, indexSnapShot + 1, this.startTimeExclusive, this.endTimeInclusive); + return new BatchMetricPoint(this.metricPoints, indexSnapShot + 1, this.startTimeExclusive, this.endTimeInclusive); } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -135,7 +135,7 @@ private void InitializeZeroTagPointIfNotInitialized() if (!this.zeroTagMetricPointInitialized) { var dt = DateTimeOffset.UtcNow; - this.metrics[0] = new MetricPoint(this.aggType, dt, null, null, this.histogramBounds); + this.metricPoints[0] = new MetricPoint(this.aggType, dt, null, null, this.histogramBounds); this.zeroTagMetricPointInitialized = true; } } @@ -202,7 +202,7 @@ private int LookupAggregatorStore(string[] tagKey, object[] tagValue, int length var seqVal = new object[length]; tagValue.CopyTo(seqVal, 0); - ref var metricPoint = ref this.metrics[aggregatorIndex]; + ref var metricPoint = ref this.metricPoints[aggregatorIndex]; var dt = DateTimeOffset.UtcNow; metricPoint = new MetricPoint(this.aggType, dt, seqKey, seqVal, this.histogramBounds); @@ -228,7 +228,7 @@ private void UpdateLong(long value, ReadOnlySpan> t return; } - this.metrics[index].Update(value); + this.metricPoints[index].Update(value); } catch (Exception) { @@ -247,7 +247,7 @@ private void UpdateLongCustomTags(long value, ReadOnlySpan Date: Fri, 8 Oct 2021 09:56:39 -0700 Subject: [PATCH 32/39] Update docs with more view configurations (#2469) --- docs/metrics/customizing-the-sdk/README.md | 145 ++++++++++++++++++--- 1 file changed, 125 insertions(+), 20 deletions(-) diff --git a/docs/metrics/customizing-the-sdk/README.md b/docs/metrics/customizing-the-sdk/README.md index deb9415a65e..4b6485d0b53 100644 --- a/docs/metrics/customizing-the-sdk/README.md +++ b/docs/metrics/customizing-the-sdk/README.md @@ -13,13 +13,12 @@ Views, etc. Naturally, almost all the customizations must be done on the Building a `MeterProvider` is done using `MeterProviderBuilder` which must be obtained by calling `Sdk.CreateMeterProviderBuilder()`. `MeterProviderBuilder` -exposes various methods which configure the provider it is going to build. -These include methods like `AddMeter`, `AddView` etc, and are explained in -subsequent sections of this document. Once configuration is done, calling -`Build()` on the `MeterProviderBuilder` builds the `MeterProvider` instance. -Once built, changes to its configuration is not allowed. In most cases, a single -`MeterProvider` is created at the application startup, and is disposed when -application shuts down. +exposes various methods which configure the provider it is going to build. These +include methods like `AddMeter`, `AddView` etc, and are explained in subsequent +sections of this document. Once configuration is done, calling `Build()` on the +`MeterProviderBuilder` builds the `MeterProvider` instance. Once built, changes +to its configuration is not allowed. In most cases, a single `MeterProvider` is +created at the application startup, and is disposed when application shuts down. The snippet below shows how to build a basic `MeterProvider`. This will create a provider with default configuration, and is not particularly useful. The @@ -38,8 +37,8 @@ using var meterProvider = Sdk.CreateMeterProviderBuilder().Build(); 1. The list of `Meter`s from which instruments are created to report measurements. -2. The list of instrumentations enabled via - [Instrumentation Library](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/glossary.md#instrumentation-library). +2. The list of instrumentations enabled via [Instrumentation + Library](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/glossary.md#instrumentation-library). 3. The list of [MetricReaders](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/metrics/sdk.md#metricreader), including exporting readers which exports metrics to @@ -88,20 +87,26 @@ using var meterProvider = Sdk.CreateMeterProviderBuilder() See [Program.cs](./Program.cs) for complete example. -**Note:** -A common mistake while configuring `MeterProvider` is forgetting to add the -required `Meter`s to the provider. It is recommended to leverage the wildcard -subscription model where it makes sense. For example, if your application is -expecting to enable instruments from a number of libraries from a company "Abc", -the you can use `AddMeter("Abc.*")` to enable all meters whose name starts with -"Abc.". +**Note:** A common mistake while configuring `MeterProvider` is forgetting to +add the required `Meter`s to the provider. It is recommended to leverage the +wildcard subscription model where it makes sense. For example, if your +application is expecting to enable instruments from a number of libraries from a +company "Abc", the you can use `AddMeter("Abc.*")` to enable all meters whose +name starts with "Abc.". ### View A [View](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/metrics/sdk.md#view) provides the ability to customize the metrics that are output by the SDK. -Following sections explains how to use this feature. +Following sections explains how to use this feature. Each section has two code +snippets. The first one uses an overload of `AddView` method that takes in the +name of the instrument as the first parameter. The `View` configuration is then +applied to the matching instrument name. The second code snippet shows how to +use an advanced selection criteria to achieve the same results. This requires +the user to provide a `Func` which offers +more flexibility in filtering the instruments to which the `View` should be +applied. #### Rename an instrument @@ -115,7 +120,19 @@ own the instrument to create it with a different name. .AddView(instrumentName: "MyCounter", name: "MyCounterRenamed") ``` -See [Program.cs](./Program.cs) for a complete example. +```csharp + // Advanced selection criteria and config via Func + .AddView((instrument) => + { + if (instrument.Meter.Name == "CompanyA.ProductB.LibraryC" && + instrument.Name == "MyCounter") + { + return new MetricStreamConfiguration() { Name = "MyCounterRenamed" }; + } + + return null; + }) +``` #### Drop an instrument @@ -133,8 +150,8 @@ then it is recommended to simply not add that `Meter` using `AddMeter`. // Advanced selection criteria and config via Func .AddView((instrument) => { - if (instrument.Meter.Name.Equals("CompanyA.ProductB.LibraryC") && - instrument.Name.Equals("MyCounterDrop")) + if (instrument.Meter.Name == "CompanyA.ProductB.LibraryC" && + instrument.Name == "MyCounterDrop") { return MetricStreamConfiguration.Drop; } @@ -143,6 +160,94 @@ then it is recommended to simply not add that `Meter` using `AddMeter`. }) ``` +#### Select specific dimensions + +When recording a measurement from an instrument, all the tags that were provided +are reported as dimensions for the given metric. Views can be used to +selectively choose a subset of dimensions to report for a given metric. This is +useful when you have a metric for which only a few of the dimensions associated +with the metric are of interest to you. + +```csharp + // Only choose "name" as the dimension for the metric "MyFruitCounter" + .AddView( + instrumentName: "MyFruitCounter", + metricStreamConfiguration: new MetricStreamConfiguration + { + TagKeys = new string[] { "name" }, + }) + + ... + // Only the dimension "name" is selected, "color" is dropped + MyFruitCounter.Add(1, new("name", "apple"), new("color", "red")); + MyFruitCounter.Add(2, new("name", "lemon"), new("color", "yellow")); + MyFruitCounter.Add(2, new("name", "apple"), new("color", "green")); + ... +``` + +```csharp + // Advanced selection criteria and config via Func + .AddView((instrument) => + { + if (instrument.Meter.Name == "CompanyA.ProductB.LibraryC" && + instrument.Name == "MyFruitCounter") + { + return new MetricStreamConfiguration + { + TagKeys = new string[] { "name" }, + }; + } + + return null; + }) +``` + +#### Specify custom bounds for Histogram + +By default, the bounds used for a Histogram are [`{ 0, 5, 10, 25, 50, 75, 100, +250, 500, 1000 }`](../../../src/OpenTelemetry/Metrics/Metric.cs#L25). Views can +be used to provide custom bounds for a Histogram. The measurements are then +aggregated against the custom bounds provided instead of the the default bounds. +This requires the use of `HistogramConfiguration`. + +```csharp + // Change Histogram bounds to count measurements under the following buckets: + // (-inf, 10] + // (10, 20] + // (20, +inf) + .AddView( + instrumentName: "MyHistogram", + new HistogramConfiguration{ BucketBounds = new double[] { 10, 20 } }) + + // If you provide an empty `double` array as `BucketBounds` to the `HistogramConfiguration`, + // the SDK will only export the sum and count for the measurements. + // There are no buckets exported in this case. + .AddView( + instrumentName: "MyHistogram", + new HistogramConfiguration { BucketBounds = new double[] { } }) +``` + +```csharp + // Advanced selection criteria and config via Func + .AddView((instrument) => + { + if (instrument.Meter.Name == "CompanyA.ProductB.LibraryC" && + instrument.Name == "MyHistogram") + { + // `HistogramConfiguration` is a child class of `MetricStreamConfiguration` + return new HistogramConfiguration + { + BucketBounds = new double[] { 10, 20 }, + }; + } + + return null; + }) +``` + +**NOTE:** The SDK currently does not support any changes to `Aggregation` type +for Views. + See [Program.cs](./Program.cs) for a complete example. ### Instrumentation From 63460be4e1d55cbd06f31c643731c99d0ec1e43f Mon Sep 17 00:00:00 2001 From: Cijo Thomas Date: Fri, 8 Oct 2021 14:06:04 -0700 Subject: [PATCH 33/39] Changelog update to release (#2472) --- src/OpenTelemetry.Api/CHANGELOG.md | 4 ++++ src/OpenTelemetry.Exporter.Console/CHANGELOG.md | 4 ++++ src/OpenTelemetry.Exporter.InMemory/CHANGELOG.md | 4 ++++ src/OpenTelemetry.Exporter.Jaeger/CHANGELOG.md | 4 ++++ .../CHANGELOG.md | 4 ++++ src/OpenTelemetry.Exporter.OpenTelemetryProtocol/CHANGELOG.md | 4 ++++ src/OpenTelemetry.Exporter.Prometheus/CHANGELOG.md | 4 ++++ src/OpenTelemetry.Exporter.ZPages/CHANGELOG.md | 4 ++++ src/OpenTelemetry.Exporter.Zipkin/CHANGELOG.md | 4 ++++ src/OpenTelemetry.Extensions.Hosting/CHANGELOG.md | 4 ++++ .../CHANGELOG.md | 4 ++++ src/OpenTelemetry.Instrumentation.AspNet/CHANGELOG.md | 4 ++++ src/OpenTelemetry.Instrumentation.AspNetCore/CHANGELOG.md | 4 ++++ src/OpenTelemetry.Instrumentation.GrpcNetClient/CHANGELOG.md | 4 ++++ src/OpenTelemetry.Instrumentation.Http/CHANGELOG.md | 4 ++++ src/OpenTelemetry.Instrumentation.SqlClient/CHANGELOG.md | 4 ++++ .../CHANGELOG.md | 4 ++++ src/OpenTelemetry.Shims.OpenTracing/CHANGELOG.md | 4 ++++ src/OpenTelemetry/CHANGELOG.md | 4 ++++ 19 files changed, 76 insertions(+) diff --git a/src/OpenTelemetry.Api/CHANGELOG.md b/src/OpenTelemetry.Api/CHANGELOG.md index fef51c6c5d7..d90fc24dfcc 100644 --- a/src/OpenTelemetry.Api/CHANGELOG.md +++ b/src/OpenTelemetry.Api/CHANGELOG.md @@ -2,6 +2,10 @@ ## Unreleased +## 1.2.0-beta1 + +Released 2021-Oct-08 + * Added `IDeferredMeterProviderBuilder` ([#2412](https://github.com/open-telemetry/opentelemetry-dotnet/pull/2412)) diff --git a/src/OpenTelemetry.Exporter.Console/CHANGELOG.md b/src/OpenTelemetry.Exporter.Console/CHANGELOG.md index 7ae3d4f1b01..ba58d824511 100644 --- a/src/OpenTelemetry.Exporter.Console/CHANGELOG.md +++ b/src/OpenTelemetry.Exporter.Console/CHANGELOG.md @@ -2,6 +2,10 @@ ## Unreleased +## 1.2.0-beta1 + +Released 2021-Oct-08 + ## 1.2.0-alpha4 Released 2021-Sep-23 diff --git a/src/OpenTelemetry.Exporter.InMemory/CHANGELOG.md b/src/OpenTelemetry.Exporter.InMemory/CHANGELOG.md index ff0a7b0c913..de5341ce855 100644 --- a/src/OpenTelemetry.Exporter.InMemory/CHANGELOG.md +++ b/src/OpenTelemetry.Exporter.InMemory/CHANGELOG.md @@ -2,6 +2,10 @@ ## Unreleased +## 1.2.0-beta1 + +Released 2021-Oct-08 + ## 1.2.0-alpha4 Released 2021-Sep-23 diff --git a/src/OpenTelemetry.Exporter.Jaeger/CHANGELOG.md b/src/OpenTelemetry.Exporter.Jaeger/CHANGELOG.md index 2813491f84d..54846046013 100644 --- a/src/OpenTelemetry.Exporter.Jaeger/CHANGELOG.md +++ b/src/OpenTelemetry.Exporter.Jaeger/CHANGELOG.md @@ -2,6 +2,10 @@ ## Unreleased +## 1.2.0-beta1 + +Released 2021-Oct-08 + ## 1.2.0-alpha4 Released 2021-Sep-23 diff --git a/src/OpenTelemetry.Exporter.OpenTelemetryProtocol.Logs/CHANGELOG.md b/src/OpenTelemetry.Exporter.OpenTelemetryProtocol.Logs/CHANGELOG.md index 1512c421622..8c033883b52 100644 --- a/src/OpenTelemetry.Exporter.OpenTelemetryProtocol.Logs/CHANGELOG.md +++ b/src/OpenTelemetry.Exporter.OpenTelemetryProtocol.Logs/CHANGELOG.md @@ -1,3 +1,7 @@ # Changelog ## Unreleased + +## 1.0.0-rc8 + +Released 2021-Oct-08 diff --git a/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/CHANGELOG.md b/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/CHANGELOG.md index 675e9209afb..3a6d5d4d230 100644 --- a/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/CHANGELOG.md +++ b/src/OpenTelemetry.Exporter.OpenTelemetryProtocol/CHANGELOG.md @@ -2,6 +2,10 @@ ## Unreleased +## 1.2.0-beta1 + +Released 2021-Oct-08 + * `MeterProviderBuilder` extension methods now support `OtlpExporterOptions` bound to `IConfiguration` when using OpenTelemetry.Extensions.Hosting ([#2413](https://github.com/open-telemetry/opentelemetry-dotnet/pull/2413)) diff --git a/src/OpenTelemetry.Exporter.Prometheus/CHANGELOG.md b/src/OpenTelemetry.Exporter.Prometheus/CHANGELOG.md index 0db52ab39ba..ff68b89701b 100644 --- a/src/OpenTelemetry.Exporter.Prometheus/CHANGELOG.md +++ b/src/OpenTelemetry.Exporter.Prometheus/CHANGELOG.md @@ -2,6 +2,10 @@ ## Unreleased +## 1.2.0-beta1 + +Released 2021-Oct-08 + ## 1.2.0-alpha4 Released 2021-Sep-23 diff --git a/src/OpenTelemetry.Exporter.ZPages/CHANGELOG.md b/src/OpenTelemetry.Exporter.ZPages/CHANGELOG.md index 7bdde8b790a..f74690c1196 100644 --- a/src/OpenTelemetry.Exporter.ZPages/CHANGELOG.md +++ b/src/OpenTelemetry.Exporter.ZPages/CHANGELOG.md @@ -2,6 +2,10 @@ ## Unreleased +## 1.0.0-rc8 + +Released 2021-Oct-08 + * Removes .NET Framework 4.5.2, 4.6 support. The minimum .NET Framework version supported is .NET 4.6.1. ([2138](https://github.com/open-telemetry/opentelemetry-dotnet/issues/2138)) diff --git a/src/OpenTelemetry.Exporter.Zipkin/CHANGELOG.md b/src/OpenTelemetry.Exporter.Zipkin/CHANGELOG.md index a135dbca212..fa628efb7e5 100644 --- a/src/OpenTelemetry.Exporter.Zipkin/CHANGELOG.md +++ b/src/OpenTelemetry.Exporter.Zipkin/CHANGELOG.md @@ -2,6 +2,10 @@ ## Unreleased +## 1.2.0-beta1 + +Released 2021-Oct-08 + * Added .NET 5.0 target and threading optimizations ([#2405](https://github.com/open-telemetry/opentelemetry-dotnet/issues/2405)) diff --git a/src/OpenTelemetry.Extensions.Hosting/CHANGELOG.md b/src/OpenTelemetry.Extensions.Hosting/CHANGELOG.md index c550963fa8b..03c76fd5a3e 100644 --- a/src/OpenTelemetry.Extensions.Hosting/CHANGELOG.md +++ b/src/OpenTelemetry.Extensions.Hosting/CHANGELOG.md @@ -2,6 +2,10 @@ ## Unreleased +## 1.0.0-rc8 + +Released 2021-Oct-08 + * Removes upper constraint for Microsoft.Extensions.Hosting.Abstractions dependency. ([#2179](https://github.com/open-telemetry/opentelemetry-dotnet/pull/2179)) diff --git a/src/OpenTelemetry.Instrumentation.AspNet.TelemetryHttpModule/CHANGELOG.md b/src/OpenTelemetry.Instrumentation.AspNet.TelemetryHttpModule/CHANGELOG.md index 88682968e35..de8c950c389 100644 --- a/src/OpenTelemetry.Instrumentation.AspNet.TelemetryHttpModule/CHANGELOG.md +++ b/src/OpenTelemetry.Instrumentation.AspNet.TelemetryHttpModule/CHANGELOG.md @@ -2,6 +2,10 @@ ## Unreleased +## 1.0.0-rc8 + +Released 2021-Oct-08 + * Adopted the donation [Microsoft.AspNet.TelemetryCorrelation](https://github.com/aspnet/Microsoft.AspNet.TelemetryCorrelation) from [.NET Foundation](https://dotnetfoundation.org/) diff --git a/src/OpenTelemetry.Instrumentation.AspNet/CHANGELOG.md b/src/OpenTelemetry.Instrumentation.AspNet/CHANGELOG.md index d8f03a0a7e4..366345e667c 100644 --- a/src/OpenTelemetry.Instrumentation.AspNet/CHANGELOG.md +++ b/src/OpenTelemetry.Instrumentation.AspNet/CHANGELOG.md @@ -2,6 +2,10 @@ ## Unreleased +## 1.0.0-rc8 + +Released 2021-Oct-08 + * Removes .NET Framework 4.5.2, .NET 4.6 support. The minimum .NET Framework version supported is .NET 4.6.1. ([#2138](https://github.com/open-telemetry/opentelemetry-dotnet/issues/2138)) diff --git a/src/OpenTelemetry.Instrumentation.AspNetCore/CHANGELOG.md b/src/OpenTelemetry.Instrumentation.AspNetCore/CHANGELOG.md index 153464c0fe4..10fb3463e61 100644 --- a/src/OpenTelemetry.Instrumentation.AspNetCore/CHANGELOG.md +++ b/src/OpenTelemetry.Instrumentation.AspNetCore/CHANGELOG.md @@ -2,6 +2,10 @@ ## Unreleased +## 1.0.0-rc8 + +Released 2021-Oct-08 + * Replaced `http.path` tag on activity with `http.target`. ([#2266](https://github.com/open-telemetry/opentelemetry-dotnet/pull/2266)) diff --git a/src/OpenTelemetry.Instrumentation.GrpcNetClient/CHANGELOG.md b/src/OpenTelemetry.Instrumentation.GrpcNetClient/CHANGELOG.md index bb21a6c0221..368d6991b54 100644 --- a/src/OpenTelemetry.Instrumentation.GrpcNetClient/CHANGELOG.md +++ b/src/OpenTelemetry.Instrumentation.GrpcNetClient/CHANGELOG.md @@ -2,6 +2,10 @@ ## Unreleased +## 1.0.0-rc8 + +Released 2021-Oct-08 + ## 1.0.0-rc7 Released 2021-Jul-12 diff --git a/src/OpenTelemetry.Instrumentation.Http/CHANGELOG.md b/src/OpenTelemetry.Instrumentation.Http/CHANGELOG.md index 53bf878be3c..b1bb87fe430 100644 --- a/src/OpenTelemetry.Instrumentation.Http/CHANGELOG.md +++ b/src/OpenTelemetry.Instrumentation.Http/CHANGELOG.md @@ -2,6 +2,10 @@ ## Unreleased +## 1.0.0-rc8 + +Released 2021-Oct-08 + * Removes .NET Framework 4.5.2 support. The minimum .NET Framework version supported is .NET 4.6.1. ([#2138](https://github.com/open-telemetry/opentelemetry-dotnet/issues/2138)) diff --git a/src/OpenTelemetry.Instrumentation.SqlClient/CHANGELOG.md b/src/OpenTelemetry.Instrumentation.SqlClient/CHANGELOG.md index fc9eda4dfbc..f30545c29f1 100644 --- a/src/OpenTelemetry.Instrumentation.SqlClient/CHANGELOG.md +++ b/src/OpenTelemetry.Instrumentation.SqlClient/CHANGELOG.md @@ -2,6 +2,10 @@ ## Unreleased +## 1.0.0-rc8 + +Released 2021-Oct-08 + * Removes .NET Framework 4.5.2 support. The minimum .NET Framework version supported is .NET 4.6.1. ([#2138](https://github.com/open-telemetry/opentelemetry-dotnet/issues/2138)) diff --git a/src/OpenTelemetry.Instrumentation.StackExchangeRedis/CHANGELOG.md b/src/OpenTelemetry.Instrumentation.StackExchangeRedis/CHANGELOG.md index fb149110925..84e6023146d 100644 --- a/src/OpenTelemetry.Instrumentation.StackExchangeRedis/CHANGELOG.md +++ b/src/OpenTelemetry.Instrumentation.StackExchangeRedis/CHANGELOG.md @@ -2,6 +2,10 @@ ## Unreleased +## 1.0.0-rc8 + +Released 2021-Oct-08 + * Adds SetVerboseDatabaseStatements option to allow setting more detailed database statement tag values. * Adds Enrich option to allow enriching activities from the source profiled command diff --git a/src/OpenTelemetry.Shims.OpenTracing/CHANGELOG.md b/src/OpenTelemetry.Shims.OpenTracing/CHANGELOG.md index 7d50303ee14..f1ec0e694e5 100644 --- a/src/OpenTelemetry.Shims.OpenTracing/CHANGELOG.md +++ b/src/OpenTelemetry.Shims.OpenTracing/CHANGELOG.md @@ -2,6 +2,10 @@ ## Unreleased +## 1.0.0-rc8 + +Released 2021-Oct-08 + * Removes .NET Framework 4.5.2 support. The minimum .NET Framework version supported is .NET 4.6.1. ([#2138](https://github.com/open-telemetry/opentelemetry-dotnet/issues/2138)) diff --git a/src/OpenTelemetry/CHANGELOG.md b/src/OpenTelemetry/CHANGELOG.md index 5bfcd8bc8ef..05cd8c96315 100644 --- a/src/OpenTelemetry/CHANGELOG.md +++ b/src/OpenTelemetry/CHANGELOG.md @@ -2,6 +2,10 @@ ## Unreleased +## 1.2.0-beta1 + +Released 2021-Oct-08 + * Exception from Observable instrument callbacks does not result in entire metrics being lost. From c79ad989fddca01a1848ca66c08171ffca52b704 Mon Sep 17 00:00:00 2001 From: yunl Date: Sat, 9 Oct 2021 10:23:16 -0700 Subject: [PATCH 34/39] Added wildcard support for meter sources. (#2459) --- src/OpenTelemetry/Metrics/MeterProviderSdk.cs | 211 ++++++++++-------- .../Metrics/MetricAPITest.cs | 83 +++++++ 2 files changed, 198 insertions(+), 96 deletions(-) diff --git a/src/OpenTelemetry/Metrics/MeterProviderSdk.cs b/src/OpenTelemetry/Metrics/MeterProviderSdk.cs index 8c12d1b62bb..e11a75697e8 100644 --- a/src/OpenTelemetry/Metrics/MeterProviderSdk.cs +++ b/src/OpenTelemetry/Metrics/MeterProviderSdk.cs @@ -19,6 +19,7 @@ using System.Diagnostics; using System.Diagnostics.Metrics; using System.Linq; +using System.Text.RegularExpressions; using OpenTelemetry.Internal; using OpenTelemetry.Resources; @@ -87,10 +88,21 @@ internal MeterProviderSdk( } // Setup Listener - var meterSourcesToSubscribe = new Dictionary(StringComparer.OrdinalIgnoreCase); - foreach (var name in meterSources) + Func shouldListenTo = instrument => false; + if (meterSources.Any(s => s.Contains('*'))) { - meterSourcesToSubscribe[name] = true; + var regex = GetWildcardRegex(meterSources); + shouldListenTo = instrument => regex.IsMatch(instrument.Meter.Name); + } + else if (meterSources.Any()) + { + var meterSourcesToSubscribe = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (var meterSource in meterSources) + { + meterSourcesToSubscribe.Add(meterSource); + } + + shouldListenTo = instrument => meterSourcesToSubscribe.Contains(instrument.Meter.Name); } this.listener = new MeterListener(); @@ -99,80 +111,82 @@ internal MeterProviderSdk( { this.listener.InstrumentPublished = (instrument, listener) => { - if (meterSourcesToSubscribe.ContainsKey(instrument.Meter.Name)) + if (!shouldListenTo(instrument)) { - // Creating list with initial capacity as the maximum - // possible size, to avoid any array resize/copy internally. - // There may be excess space wasted, but it'll eligible for - // GC right after this method. - var metricStreamConfigs = new List(viewConfigCount); - foreach (var viewConfig in this.viewConfigs) - { - var metricStreamConfig = viewConfig(instrument); - if (metricStreamConfig != null) - { - metricStreamConfigs.Add(metricStreamConfig); - } - } + return; + } - if (metricStreamConfigs.Count == 0) + // Creating list with initial capacity as the maximum + // possible size, to avoid any array resize/copy internally. + // There may be excess space wasted, but it'll eligible for + // GC right after this method. + var metricStreamConfigs = new List(viewConfigCount); + foreach (var viewConfig in this.viewConfigs) + { + var metricStreamConfig = viewConfig(instrument); + if (metricStreamConfig != null) { - // No views matched. Add null - // which will apply defaults. - // Users can turn off this default - // by adding a view like below as the last view. - // .AddView(instrumentName: "*", new MetricStreamConfiguration() { Aggregation = Aggregation.Drop }) - metricStreamConfigs.Add(null); + metricStreamConfigs.Add(metricStreamConfig); } + } - var maxCountMetricsToBeCreated = metricStreamConfigs.Count; + if (metricStreamConfigs.Count == 0) + { + // No views matched. Add null + // which will apply defaults. + // Users can turn off this default + // by adding a view like below as the last view. + // .AddView(instrumentName: "*", new MetricStreamConfiguration() { Aggregation = Aggregation.Drop }) + metricStreamConfigs.Add(null); + } - // Create list with initial capacity as the max metric count. - // Due to duplicate/max limit, we may not end up using them - // all, and that memory is wasted until Meter disposed. - // TODO: Revisit to see if we need to do metrics.TrimExcess() - var metrics = new List(maxCountMetricsToBeCreated); - lock (this.instrumentCreationLock) + var maxCountMetricsToBeCreated = metricStreamConfigs.Count; + + // Create list with initial capacity as the max metric count. + // Due to duplicate/max limit, we may not end up using them + // all, and that memory is wasted until Meter disposed. + // TODO: Revisit to see if we need to do metrics.TrimExcess() + var metrics = new List(maxCountMetricsToBeCreated); + lock (this.instrumentCreationLock) + { + for (int i = 0; i < maxCountMetricsToBeCreated; i++) { - for (int i = 0; i < maxCountMetricsToBeCreated; i++) + var metricStreamConfig = metricStreamConfigs[i]; + var metricStreamName = metricStreamConfig?.Name ?? instrument.Name; + if (this.metricStreamNames.ContainsKey(metricStreamName)) + { + // TODO: Log that instrument is ignored + // as the resulting Metric name is conflicting + // with existing name. + continue; + } + + if (metricStreamConfig?.Aggregation == Aggregation.Drop) { - var metricStreamConfig = metricStreamConfigs[i]; - var metricStreamName = metricStreamConfig?.Name ?? instrument.Name; - if (this.metricStreamNames.ContainsKey(metricStreamName)) - { - // TODO: Log that instrument is ignored - // as the resulting Metric name is conflicting - // with existing name. - continue; - } - - if (metricStreamConfig?.Aggregation == Aggregation.Drop) - { - // TODO: Log that instrument is ignored - // as user explicitly asked to drop it - // with View. - continue; - } - - var index = ++this.metricIndex; - if (index >= MaxMetrics) - { - // TODO: Log that instrument is ignored - // as max number of Metrics have reached. - } - else - { - Metric metric; - var metricDescription = metricStreamConfig?.Description ?? instrument.Description; - string[] tagKeysInteresting = metricStreamConfig?.TagKeys; - double[] histogramBucketBounds = (metricStreamConfig is HistogramConfiguration histogramConfig - && histogramConfig.BucketBounds != null) ? histogramConfig.BucketBounds : null; - metric = new Metric(instrument, temporality, metricStreamName, metricDescription, histogramBucketBounds, tagKeysInteresting); - - this.metrics[index] = metric; - metrics.Add(metric); - this.metricStreamNames.Add(metricStreamName, true); - } + // TODO: Log that instrument is ignored + // as user explicitly asked to drop it + // with View. + continue; + } + + var index = ++this.metricIndex; + if (index >= MaxMetrics) + { + // TODO: Log that instrument is ignored + // as max number of Metrics have reached. + } + else + { + Metric metric; + var metricDescription = metricStreamConfig?.Description ?? instrument.Description; + string[] tagKeysInteresting = metricStreamConfig?.TagKeys; + double[] histogramBucketBounds = (metricStreamConfig is HistogramConfiguration histogramConfig + && histogramConfig.BucketBounds != null) ? histogramConfig.BucketBounds : null; + metric = new Metric(instrument, temporality, metricStreamName, metricDescription, histogramBucketBounds, tagKeysInteresting); + + this.metrics[index] = metric; + metrics.Add(metric); + this.metricStreamNames.Add(metricStreamName, true); } } @@ -197,40 +211,39 @@ internal MeterProviderSdk( { this.listener.InstrumentPublished = (instrument, listener) => { + if (!shouldListenTo(instrument)) + { + OpenTelemetrySdkEventSource.Log.MetricInstrumentIgnored(instrument.Name, instrument.Meter.Name, "Instrument belongs to a Meter not subscribed by the provider.", "Use AddMeter to add the Meter to the provider."); + return; + } + try { - if (meterSourcesToSubscribe.ContainsKey(instrument.Meter.Name)) + var metricName = instrument.Name; + Metric metric = null; + lock (this.instrumentCreationLock) { - var metricName = instrument.Name; - Metric metric = null; - lock (this.instrumentCreationLock) + if (this.metricStreamNames.ContainsKey(metricName)) { - if (this.metricStreamNames.ContainsKey(metricName)) - { - OpenTelemetrySdkEventSource.Log.MetricInstrumentIgnored(metricName, instrument.Meter.Name, "Metric name conflicting with existing name.", "Either change the name of the instrument or change name using View."); - return; - } - - var index = ++this.metricIndex; - if (index >= MaxMetrics) - { - OpenTelemetrySdkEventSource.Log.MetricInstrumentIgnored(metricName, instrument.Meter.Name, "Maximum allowed Metrics for the provider exceeded.", "Use views to drop unused instruments. Or configure Provider to allow higher limit."); - return; - } - else - { - metric = new Metric(instrument, temporality, metricName, instrument.Description); - this.metrics[index] = metric; - this.metricStreamNames.Add(metricName, true); - } + OpenTelemetrySdkEventSource.Log.MetricInstrumentIgnored(metricName, instrument.Meter.Name, "Metric name conflicting with existing name.", "Either change the name of the instrument or change name using View."); + return; } - listener.EnableMeasurementEvents(instrument, metric); - } - else - { - OpenTelemetrySdkEventSource.Log.MetricInstrumentIgnored(instrument.Name, instrument.Meter.Name, "Instrument belongs to a Meter not subscribed by the provider.", "Use AddMeter to add the Meter to the provider."); + var index = ++this.metricIndex; + if (index >= MaxMetrics) + { + OpenTelemetrySdkEventSource.Log.MetricInstrumentIgnored(metricName, instrument.Meter.Name, "Maximum allowed Metrics for the provider exceeded.", "Use views to drop unused instruments. Or configure Provider to allow higher limit."); + return; + } + else + { + metric = new Metric(instrument, temporality, metricName, instrument.Description); + this.metrics[index] = metric; + this.metricStreamNames.Add(metricName, true); + } } + + listener.EnableMeasurementEvents(instrument, metric); } catch (Exception) { @@ -251,6 +264,12 @@ internal MeterProviderSdk( this.listener.MeasurementsCompleted = (instrument, state) => this.MeasurementsCompleted(instrument, state); this.listener.Start(); + + static Regex GetWildcardRegex(IEnumerable collection) + { + var pattern = '^' + string.Join("|", from name in collection select "(?:" + Regex.Escape(name).Replace("\\*", ".*") + ')') + '$'; + return new Regex(pattern, RegexOptions.Compiled | RegexOptions.IgnoreCase); + } } internal Resource Resource { get; } diff --git a/test/OpenTelemetry.Tests/Metrics/MetricAPITest.cs b/test/OpenTelemetry.Tests/Metrics/MetricAPITest.cs index 3bd95500084..73dcb3df392 100644 --- a/test/OpenTelemetry.Tests/Metrics/MetricAPITest.cs +++ b/test/OpenTelemetry.Tests/Metrics/MetricAPITest.cs @@ -165,6 +165,89 @@ void ProcessExport(Batch batch) Assert.Equal(1, metricCount); } + [Theory] + [InlineData(true)] + [InlineData(false)] + public void MeterSourcesWildcardSupportMatchTest(bool hasView) + { + var meterNames = new[] + { + "AbcCompany.XyzProduct.ComponentA", + "abcCompany.xYzProduct.componentC", // Wildcard match is case insensitive. + "DefCompany.AbcProduct.ComponentC", + "DefCompany.XyzProduct.ComponentC", // Wildcard match supports matching multiple patterns. + "GhiCompany.qweProduct.ComponentN", + "SomeCompany.SomeProduct.SomeComponent", + }; + + using var meter1 = new Meter(meterNames[0]); + using var meter2 = new Meter(meterNames[1]); + using var meter3 = new Meter(meterNames[2]); + using var meter4 = new Meter(meterNames[3]); + using var meter5 = new Meter(meterNames[4]); + using var meter6 = new Meter(meterNames[5]); + + var exportedItems = new List(); + var meterProviderBuilder = Sdk.CreateMeterProviderBuilder() + .AddMeter("AbcCompany.XyzProduct.*") + .AddMeter("DefCompany.*.ComponentC") + .AddMeter("GhiCompany.qweProduct.ComponentN") // Mixing of non-wildcard meter name and wildcard meter name. + .AddInMemoryExporter(exportedItems); + + if (hasView) + { + meterProviderBuilder.AddView("myGauge1", "newName"); + } + + using var meterProvider = meterProviderBuilder.Build(); + + var measurement = new Measurement(100, new("name", "apple"), new("color", "red")); + meter1.CreateObservableGauge("myGauge1", () => measurement); + meter2.CreateObservableGauge("myGauge2", () => measurement); + meter3.CreateObservableGauge("myGauge3", () => measurement); + meter4.CreateObservableGauge("myGauge4", () => measurement); + meter5.CreateObservableGauge("myGauge5", () => measurement); + meter6.CreateObservableGauge("myGauge6", () => measurement); + + meterProvider.ForceFlush(MaxTimeToAllowForFlush); + + Assert.True(exportedItems.Count == 5); // "SomeCompany.SomeProduct.SomeComponent" will not be subscribed. + + if (hasView) + { + Assert.Equal("newName", exportedItems[0].Name); + } + else + { + Assert.Equal("myGauge1", exportedItems[0].Name); + } + + Assert.Equal("myGauge2", exportedItems[1].Name); + Assert.Equal("myGauge3", exportedItems[2].Name); + Assert.Equal("myGauge4", exportedItems[3].Name); + Assert.Equal("myGauge5", exportedItems[4].Name); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public void MeterSourcesWildcardSupportWithoutAddingMeterToProvider(bool hasView) + { + var exportedItems = new List(); + var meterProviderBuilder = Sdk.CreateMeterProviderBuilder() + .AddInMemoryExporter(exportedItems); + + if (hasView) + { + meterProviderBuilder.AddView("gauge1", "renamed"); + } + + using var meterProvider = meterProviderBuilder.Build(); + var measurement = new Measurement(100, new("name", "apple"), new("color", "red")); + meterProvider.ForceFlush(MaxTimeToAllowForFlush); + Assert.True(exportedItems.Count == 0); + } + [Theory] [InlineData(true)] [InlineData(false)] From 4bc68a74dafc168811ff237451191df4a2a0a6f3 Mon Sep 17 00:00:00 2001 From: yunl Date: Mon, 11 Oct 2021 15:55:07 -0700 Subject: [PATCH 35/39] Changes around MeterProviderSdk, resolving pending comments. (#2474) --- src/OpenTelemetry/Metrics/MeterProviderSdk.cs | 1 + test/OpenTelemetry.Tests/Metrics/MetricAPITest.cs | 15 ++++++++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/OpenTelemetry/Metrics/MeterProviderSdk.cs b/src/OpenTelemetry/Metrics/MeterProviderSdk.cs index e11a75697e8..d3e61961e5c 100644 --- a/src/OpenTelemetry/Metrics/MeterProviderSdk.cs +++ b/src/OpenTelemetry/Metrics/MeterProviderSdk.cs @@ -113,6 +113,7 @@ internal MeterProviderSdk( { if (!shouldListenTo(instrument)) { + OpenTelemetrySdkEventSource.Log.MetricInstrumentIgnored(instrument.Name, instrument.Meter.Name, "Instrument belongs to a Meter not subscribed by the provider.", "Use AddMeter to add the Meter to the provider."); return; } diff --git a/test/OpenTelemetry.Tests/Metrics/MetricAPITest.cs b/test/OpenTelemetry.Tests/Metrics/MetricAPITest.cs index 73dcb3df392..08d45e032d4 100644 --- a/test/OpenTelemetry.Tests/Metrics/MetricAPITest.cs +++ b/test/OpenTelemetry.Tests/Metrics/MetricAPITest.cs @@ -231,8 +231,17 @@ public void MeterSourcesWildcardSupportMatchTest(bool hasView) [Theory] [InlineData(true)] [InlineData(false)] - public void MeterSourcesWildcardSupportWithoutAddingMeterToProvider(bool hasView) + public void MeterSourcesWildcardSupportNegativeTestNoMeterAdded(bool hasView) { + var meterNames = new[] + { + "AbcCompany.XyzProduct.ComponentA", + "abcCompany.xYzProduct.componentC", + }; + + using var meter1 = new Meter(meterNames[0]); + using var meter2 = new Meter(meterNames[1]); + var exportedItems = new List(); var meterProviderBuilder = Sdk.CreateMeterProviderBuilder() .AddInMemoryExporter(exportedItems); @@ -244,6 +253,10 @@ public void MeterSourcesWildcardSupportWithoutAddingMeterToProvider(bool hasView using var meterProvider = meterProviderBuilder.Build(); var measurement = new Measurement(100, new("name", "apple"), new("color", "red")); + + meter1.CreateObservableGauge("myGauge1", () => measurement); + meter2.CreateObservableGauge("myGauge2", () => measurement); + meterProvider.ForceFlush(MaxTimeToAllowForFlush); Assert.True(exportedItems.Count == 0); } From cfa01ba1b35988bb16773fbf0937dc413f989b66 Mon Sep 17 00:00:00 2001 From: Utkarsh Umesan Pillai Date: Mon, 11 Oct 2021 17:02:15 -0700 Subject: [PATCH 36/39] Update docs for views (#2475) --- docs/metrics/customizing-the-sdk/README.md | 27 ++++++++++++++++++---- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/docs/metrics/customizing-the-sdk/README.md b/docs/metrics/customizing-the-sdk/README.md index 4b6485d0b53..2fcc67b921f 100644 --- a/docs/metrics/customizing-the-sdk/README.md +++ b/docs/metrics/customizing-the-sdk/README.md @@ -160,7 +160,7 @@ then it is recommended to simply not add that `Meter` using `AddMeter`. }) ``` -#### Select specific dimensions +#### Select specific tags When recording a measurement from an instrument, all the tags that were provided are reported as dimensions for the given metric. Views can be used to @@ -183,6 +183,22 @@ with the metric are of interest to you. MyFruitCounter.Add(2, new("name", "lemon"), new("color", "yellow")); MyFruitCounter.Add(2, new("name", "apple"), new("color", "green")); ... + + // If you provide an empty `string` array as `TagKeys` to the `MetricStreamConfiguration` + // the SDK will drop all the dimensions associated with the metric + .AddView( + instrumentName: "MyFruitCounter", + metricStreamConfiguration: new MetricStreamConfiguration + { + TagKeys = new string[] { }, + }) + + ... + // both "name" and "color" are dropped + MyFruitCounter.Add(1, new("name", "apple"), new("color", "red")); + MyFruitCounter.Add(2, new("name", "lemon"), new("color", "yellow")); + MyFruitCounter.Add(2, new("name", "apple"), new("color", "green")); + ... ``` ```csharp @@ -205,10 +221,11 @@ with the metric are of interest to you. #### Specify custom bounds for Histogram By default, the bounds used for a Histogram are [`{ 0, 5, 10, 25, 50, 75, 100, -250, 500, 1000 }`](../../../src/OpenTelemetry/Metrics/Metric.cs#L25). Views can -be used to provide custom bounds for a Histogram. The measurements are then -aggregated against the custom bounds provided instead of the the default bounds. -This requires the use of `HistogramConfiguration`. +250, 500, +1000}`](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/metrics/sdk.md#explicit-bucket-histogram-aggregation). +Views can be used to provide custom bounds for a Histogram. The measurements are +then aggregated using the custom bounds provided instead of the the default +bounds. This requires the use of `HistogramConfiguration`. ```csharp // Change Histogram bounds to count measurements under the following buckets: From ed814d1d6d9b9c9bda7aebca5b7a09fcadaa1ec8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robert=20Paj=C4=85k?= Date: Tue, 12 Oct 2021 18:59:19 +0200 Subject: [PATCH 37/39] Throw an exception when parsing an OTEL_BSP_* env var fails (#2395) --- .../BatchExportActivityProcessorOptions.cs | 21 +++++++++++-------- ...BatchExportActivityProcessorOptionsTest.cs | 4 +--- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/src/OpenTelemetry/Trace/BatchExportActivityProcessorOptions.cs b/src/OpenTelemetry/Trace/BatchExportActivityProcessorOptions.cs index a150cc74d8e..d0dca246edc 100644 --- a/src/OpenTelemetry/Trace/BatchExportActivityProcessorOptions.cs +++ b/src/OpenTelemetry/Trace/BatchExportActivityProcessorOptions.cs @@ -21,6 +21,11 @@ namespace OpenTelemetry.Trace { + /// + /// Batch span processor options. + /// OTEL_BSP_MAX_QUEUE_SIZE, OTEL_BSP_MAX_EXPORT_BATCH_SIZE, OTEL_BSP_EXPORT_TIMEOUT, OTEL_BSP_SCHEDULE_DELAY + /// environment variables are parsed during object construction. + /// public class BatchExportActivityProcessorOptions : BatchExportProcessorOptions { internal const string MaxQueueSizeEnvVarKey = "OTEL_BSP_MAX_QUEUE_SIZE"; @@ -35,28 +40,28 @@ public BatchExportActivityProcessorOptions() { int value; - if (TryLoadEnvVarInt(ExporterTimeoutEnvVarKey, out value)) + if (LoadEnvVarInt(ExporterTimeoutEnvVarKey, out value)) { this.ExporterTimeoutMilliseconds = value; } - if (TryLoadEnvVarInt(MaxExportBatchSizeEnvVarKey, out value)) + if (LoadEnvVarInt(MaxExportBatchSizeEnvVarKey, out value)) { this.MaxExportBatchSize = value; } - if (TryLoadEnvVarInt(MaxQueueSizeEnvVarKey, out value)) + if (LoadEnvVarInt(MaxQueueSizeEnvVarKey, out value)) { this.MaxQueueSize = value; } - if (TryLoadEnvVarInt(ScheduledDelayEnvVarKey, out value)) + if (LoadEnvVarInt(ScheduledDelayEnvVarKey, out value)) { this.ScheduledDelayMilliseconds = value; } } - private static bool TryLoadEnvVarInt(string envVarKey, out int result) + private static bool LoadEnvVarInt(string envVarKey, out int result) { result = 0; @@ -78,13 +83,11 @@ private static bool TryLoadEnvVarInt(string envVarKey, out int result) return false; } - if (!int.TryParse(value, out var parsedValue)) + if (!int.TryParse(value, out result)) { - OpenTelemetrySdkEventSource.Log.FailedToParseEnvironmentVariable(envVarKey, value); - return false; + throw new ArgumentException($"{envVarKey} environment variable has an invalid value: '${value}'"); } - result = parsedValue; return true; } } diff --git a/test/OpenTelemetry.Tests/Trace/BatchExportActivityProcessorOptionsTest.cs b/test/OpenTelemetry.Tests/Trace/BatchExportActivityProcessorOptionsTest.cs index 327752fa316..cd62ae57c7e 100644 --- a/test/OpenTelemetry.Tests/Trace/BatchExportActivityProcessorOptionsTest.cs +++ b/test/OpenTelemetry.Tests/Trace/BatchExportActivityProcessorOptionsTest.cs @@ -63,9 +63,7 @@ public void BatchExportProcessorOptions_InvalidPortEnvironmentVariableOverride() { Environment.SetEnvironmentVariable(BatchExportActivityProcessorOptions.ExporterTimeoutEnvVarKey, "invalid"); - var options = new BatchExportActivityProcessorOptions(); - - Assert.Equal(30000, options.ExporterTimeoutMilliseconds); // use default + Assert.Throws(() => new BatchExportActivityProcessorOptions()); } [Fact] From 6664781c765afda68f25acbfef69bf0a0f7a57d5 Mon Sep 17 00:00:00 2001 From: Cijo Thomas Date: Tue, 12 Oct 2021 15:50:42 -0700 Subject: [PATCH 38/39] Add OTLP LogExporter example (#2481) --- examples/Console/Examples.Console.csproj | 1 + examples/Console/Program.cs | 10 ++- examples/Console/TestLogs.cs | 79 +++++++++++++++++++ .../otlp-collector-example/config.yaml | 3 + 4 files changed, 92 insertions(+), 1 deletion(-) create mode 100644 examples/Console/TestLogs.cs diff --git a/examples/Console/Examples.Console.csproj b/examples/Console/Examples.Console.csproj index 84ecf4f5b71..e2eaf19f524 100644 --- a/examples/Console/Examples.Console.csproj +++ b/examples/Console/Examples.Console.csproj @@ -37,5 +37,6 @@ + diff --git a/examples/Console/Program.cs b/examples/Console/Program.cs index 7a40dbe168e..afebf22d8a4 100644 --- a/examples/Console/Program.cs +++ b/examples/Console/Program.cs @@ -43,12 +43,13 @@ public class Program /// Arguments from command line. public static void Main(string[] args) { - Parser.Default.ParseArguments(args) + Parser.Default.ParseArguments(args) .MapResult( (JaegerOptions options) => TestJaegerExporter.Run(options.Host, options.Port), (ZipkinOptions options) => TestZipkinExporter.Run(options.Uri), (PrometheusOptions options) => TestPrometheusExporter.Run(options.Port, options.DurationInMins), (MetricsOptions options) => TestMetrics.Run(options), + (LogsOptions options) => TestLogs.Run(options), (GrpcNetClientOptions options) => TestGrpcNetClient.Run(), (HttpClientOptions options) => TestHttpClient.Run(), (RedisOptions options) => TestRedis.Run(options.Uri), @@ -169,6 +170,13 @@ internal class OtlpOptions public string Protocol { get; set; } } + [Verb("logs", HelpText = "Specify the options required to test Logs")] + internal class LogsOptions + { + [Option("useExporter", Default = "otlp", HelpText = "Options include otlp or console.", Required = false)] + public string UseExporter { get; set; } + } + [Verb("inmemory", HelpText = "Specify the options required to test InMemory Exporter")] internal class InMemoryOptions { diff --git a/examples/Console/TestLogs.cs b/examples/Console/TestLogs.cs new file mode 100644 index 00000000000..aeb405c8b89 --- /dev/null +++ b/examples/Console/TestLogs.cs @@ -0,0 +1,79 @@ +// +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using OpenTelemetry; +using OpenTelemetry.Logs; + +namespace Examples.Console +{ + internal class TestLogs + { + internal static object Run(LogsOptions options) + { + using var loggerFactory = LoggerFactory.Create(builder => + { + builder.AddOpenTelemetry((opt) => + { + opt.IncludeFormattedMessage = true; + if (options.UseExporter.ToLower() == "otlp") + { + /* + * Prerequisite to run this example: + * Set up an OpenTelemetry Collector to run on local docker. + * + * Open a terminal window at the examples/Console/ directory and + * launch the OpenTelemetry Collector with an OTLP receiver, by running: + * + * - On Unix based systems use: + * docker run --rm -it -p 4317:4317 -v $(pwd):/cfg otel/opentelemetry-collector:0.33.0 --config=/cfg/otlp-collector-example/config.yaml + * + * - On Windows use: + * docker run --rm -it -p 4317:4317 -v "%cd%":/cfg otel/opentelemetry-collector:0.33.0 --config=/cfg/otlp-collector-example/config.yaml + * + * Open another terminal window at the examples/Console/ directory and + * launch the OTLP example by running: + * + * dotnet run logs --useExporter otlp + * + * The OpenTelemetry Collector will output all received metrics to the stdout of its terminal. + * + */ + + // Adding the OtlpExporter creates a GrpcChannel. + // This switch must be set before creating a GrpcChannel/HttpClient when calling an insecure gRPC service. + // See: https://docs.microsoft.com/aspnet/core/grpc/troubleshoot#call-insecure-grpc-services-with-net-core-client + AppContext.SetSwitch("System.Net.Http.SocketsHttpHandler.Http2UnencryptedSupport", true); + + opt.AddOtlpExporter(); + } + else + { + opt.AddConsoleExporter(); + } + }); + }); + + var logger = loggerFactory.CreateLogger(); + logger.LogInformation("Hello from {name} {price}.", "tomato", 2.99); + return null; + } + } +} diff --git a/examples/Console/otlp-collector-example/config.yaml b/examples/Console/otlp-collector-example/config.yaml index 558ecf8423b..10b24b00a3d 100644 --- a/examples/Console/otlp-collector-example/config.yaml +++ b/examples/Console/otlp-collector-example/config.yaml @@ -21,3 +21,6 @@ service: metrics: receivers: [otlp] exporters: [logging] + logs: + receivers: [otlp] + exporters: [logging] From 90d2906eda2d77c9814b22d57c4a5c08e5825146 Mon Sep 17 00:00:00 2001 From: Reiley Yang Date: Tue, 12 Oct 2021 16:49:58 -0700 Subject: [PATCH 39/39] Fix PeriodicExportingMetricReader timing (#2477) --- .../Metrics/PeriodicExportingMetricReader.cs | 94 ++++++++++++------- 1 file changed, 61 insertions(+), 33 deletions(-) diff --git a/src/OpenTelemetry/Metrics/PeriodicExportingMetricReader.cs b/src/OpenTelemetry/Metrics/PeriodicExportingMetricReader.cs index 54b5a6dcc4c..22e8bb64381 100644 --- a/src/OpenTelemetry/Metrics/PeriodicExportingMetricReader.cs +++ b/src/OpenTelemetry/Metrics/PeriodicExportingMetricReader.cs @@ -15,8 +15,8 @@ // using System; +using System.Diagnostics; using System.Threading; -using System.Threading.Tasks; namespace OpenTelemetry.Metrics { @@ -25,9 +25,11 @@ public class PeriodicExportingMetricReader : BaseExportingMetricReader internal const int DefaultExportIntervalMilliseconds = 60000; internal const int DefaultExportTimeoutMilliseconds = 30000; - private readonly Task exportTask; - private readonly CancellationTokenSource token; - private bool disposed; + private readonly int exportIntervalMilliseconds; + private readonly int exportTimeoutMilliseconds; + private readonly Thread exporterThread; + private readonly AutoResetEvent exportTrigger = new AutoResetEvent(false); + private readonly ManualResetEvent shutdownTrigger = new ManualResetEvent(false); public PeriodicExportingMetricReader( BaseExporter exporter, @@ -35,52 +37,78 @@ public PeriodicExportingMetricReader( int exportTimeoutMilliseconds = DefaultExportTimeoutMilliseconds) : base(exporter) { + if (exportIntervalMilliseconds <= 0) + { + throw new ArgumentOutOfRangeException(nameof(exportIntervalMilliseconds), exportIntervalMilliseconds, "exportIntervalMilliseconds should be greater than zero."); + } + + if (exportTimeoutMilliseconds < 0) + { + throw new ArgumentOutOfRangeException(nameof(exportTimeoutMilliseconds), exportTimeoutMilliseconds, "exportTimeoutMilliseconds should be non-negative."); + } + if ((this.SupportedExportModes & ExportModes.Push) != ExportModes.Push) { throw new InvalidOperationException("The exporter does not support push mode."); } - this.token = new CancellationTokenSource(); + this.exportIntervalMilliseconds = exportIntervalMilliseconds; + this.exportTimeoutMilliseconds = exportTimeoutMilliseconds; - // TODO: Use dedicated thread. - this.exportTask = new Task(() => + this.exporterThread = new Thread(new ThreadStart(this.ExporterProc)) { - while (!this.token.IsCancellationRequested) - { - // TODO: Should pass CancellationToken - Task.Delay(exportIntervalMilliseconds).Wait(); - this.Collect(); - } - }); - - this.exportTask.Start(); + IsBackground = true, + Name = $"OpenTelemetry-{nameof(PeriodicExportingMetricReader)}-{exporter.GetType().Name}", + }; + this.exporterThread.Start(); } - /// - protected override void Dispose(bool disposing) + /// + protected override bool OnShutdown(int timeoutMilliseconds) { - if (this.disposed) + var result = true; + + this.shutdownTrigger.Set(); + + if (timeoutMilliseconds == Timeout.Infinite) { - return; + this.exporterThread.Join(); + result = this.exporter.Shutdown() && result; } + else + { + var sw = Stopwatch.StartNew(); + result = this.exporterThread.Join(timeoutMilliseconds) && result; + var timeout = timeoutMilliseconds - sw.ElapsedMilliseconds; + result = this.exporter.Shutdown((int)Math.Max(timeout, 0)) && result; + } + + return result; + } + + private void ExporterProc() + { + var sw = Stopwatch.StartNew(); + var triggers = new WaitHandle[] { this.exportTrigger, this.shutdownTrigger }; - if (disposing) + while (true) { - try - { - this.token.Cancel(); - this.exportTask.Wait(); - this.token.Dispose(); - } - catch (Exception) + var timeout = (int)(this.exportIntervalMilliseconds - (sw.ElapsedMilliseconds % this.exportIntervalMilliseconds)); + var index = WaitHandle.WaitAny(triggers, timeout); + + switch (index) { - // TODO: Log + case 0: // export + this.Collect(this.exportTimeoutMilliseconds); + break; + case 1: // shutdown + this.Collect(this.exportTimeoutMilliseconds); // TODO: do we want to use the shutdown timeout here? + return; + case WaitHandle.WaitTimeout: // timer + this.Collect(this.exportTimeoutMilliseconds); + break; } } - - this.disposed = true; - - base.Dispose(disposing); } } }