From df0e7a4addec31fd59c83d64f5ad18f60bdc945c Mon Sep 17 00:00:00 2001 From: joegoldman2 <147369450+joegoldman@users.noreply.github.com> Date: Sat, 27 Apr 2024 08:55:03 +0000 Subject: [PATCH] Add Kubernetes support in Container Resource Detector --- opentelemetry-dotnet-contrib.sln | 1 + ...OpenTelemetry.ResourceDetectors.AWS.csproj | 1 + .../ContainerExtensionsEventSource.cs | 29 ---- .../ContainerResourceDetector.cs | 152 +++++++++++++++--- .../ContainerResourceEventSource.cs | 59 +++++++ .../Models/K8sContainerStatus.cs | 15 ++ .../Models/K8sPod.cs | 12 ++ .../Models/K8sPodStatus.cs | 13 ++ ...lemetry.ResourceDetectors.Container.csproj | 4 + .../SourceGenerationContext.cs | 24 +++ .../ResourceDetectorUtils.cs | 2 +- ...y.ResourceDetectors.Container.Tests.csproj | 4 - 12 files changed, 256 insertions(+), 60 deletions(-) delete mode 100644 src/OpenTelemetry.ResourceDetectors.Container/ContainerExtensionsEventSource.cs create mode 100644 src/OpenTelemetry.ResourceDetectors.Container/ContainerResourceEventSource.cs create mode 100644 src/OpenTelemetry.ResourceDetectors.Container/Models/K8sContainerStatus.cs create mode 100644 src/OpenTelemetry.ResourceDetectors.Container/Models/K8sPod.cs create mode 100644 src/OpenTelemetry.ResourceDetectors.Container/Models/K8sPodStatus.cs create mode 100644 src/OpenTelemetry.ResourceDetectors.Container/SourceGenerationContext.cs rename src/{OpenTelemetry.ResourceDetectors.AWS => Shared}/ResourceDetectorUtils.cs (98%) diff --git a/opentelemetry-dotnet-contrib.sln b/opentelemetry-dotnet-contrib.sln index a49632c03c..83965ed134 100644 --- a/opentelemetry-dotnet-contrib.sln +++ b/opentelemetry-dotnet-contrib.sln @@ -284,6 +284,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Shared", "Shared", "{1FCC8E src\Shared\PropertyFetcher.AOT.cs = src\Shared\PropertyFetcher.AOT.cs src\Shared\PropertyFetcher.cs = src\Shared\PropertyFetcher.cs src\Shared\RedactionHelper.cs = src\Shared\RedactionHelper.cs + src\Shared\ResourceDetectorUtils.cs = src\Shared\ResourceDetectorUtils.cs src\Shared\ResourceSemanticConventions.cs = src\Shared\ResourceSemanticConventions.cs src\Shared\SemanticConventions.cs = src\Shared\SemanticConventions.cs src\Shared\ServerCertificateValidationHandler.cs = src\Shared\ServerCertificateValidationHandler.cs diff --git a/src/OpenTelemetry.ResourceDetectors.AWS/OpenTelemetry.ResourceDetectors.AWS.csproj b/src/OpenTelemetry.ResourceDetectors.AWS/OpenTelemetry.ResourceDetectors.AWS.csproj index 0cec692434..d518de345b 100644 --- a/src/OpenTelemetry.ResourceDetectors.AWS/OpenTelemetry.ResourceDetectors.AWS.csproj +++ b/src/OpenTelemetry.ResourceDetectors.AWS/OpenTelemetry.ResourceDetectors.AWS.csproj @@ -24,6 +24,7 @@ + diff --git a/src/OpenTelemetry.ResourceDetectors.Container/ContainerExtensionsEventSource.cs b/src/OpenTelemetry.ResourceDetectors.Container/ContainerExtensionsEventSource.cs deleted file mode 100644 index 2eac9e80b2..0000000000 --- a/src/OpenTelemetry.ResourceDetectors.Container/ContainerExtensionsEventSource.cs +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright The OpenTelemetry Authors -// SPDX-License-Identifier: Apache-2.0 - -using System; -using System.Diagnostics.Tracing; -using OpenTelemetry.Internal; - -namespace OpenTelemetry.ResourceDetectors.Container; - -[EventSource(Name = "OpenTelemetry-ResourceDetectors-Container")] -internal class ContainerExtensionsEventSource : EventSource -{ - public static ContainerExtensionsEventSource Log = new(); - - [NonEvent] - public void ExtractResourceAttributesException(string format, Exception ex) - { - if (this.IsEnabled(EventLevel.Error, (EventKeywords)(-1))) - { - this.FailedToExtractResourceAttributes(format, ex.ToInvariantString()); - } - } - - [Event(1, Message = "Failed to extract resource attributes in '{0}'.", Level = EventLevel.Error)] - public void FailedToExtractResourceAttributes(string format, string exception) - { - this.WriteEvent(1, format, exception); - } -} diff --git a/src/OpenTelemetry.ResourceDetectors.Container/ContainerResourceDetector.cs b/src/OpenTelemetry.ResourceDetectors.Container/ContainerResourceDetector.cs index ec7d1705d3..ed4c2fd1c9 100644 --- a/src/OpenTelemetry.ResourceDetectors.Container/ContainerResourceDetector.cs +++ b/src/OpenTelemetry.ResourceDetectors.Container/ContainerResourceDetector.cs @@ -4,7 +4,10 @@ using System; using System.Collections.Generic; using System.IO; +using System.Linq; +using System.Text; using System.Text.RegularExpressions; +using OpenTelemetry.ResourceDetectors.Container.Models; using OpenTelemetry.ResourceDetectors.Container.Utils; using OpenTelemetry.Resources; @@ -18,6 +21,13 @@ public class ContainerResourceDetector : IResourceDetector private const string Filepath = "/proc/self/cgroup"; private const string FilepathV2 = "/proc/self/mountinfo"; private const string Hostname = "hostname"; + private const string K8sServiceHostKey = "KUBERNETES_SERVICE_HOST"; + private const string K8sServicePortKey = "KUBERNETES_SERVICE_PORT_HTTPS"; + private const string K8sNamespaceKey = "KUBERNETES_NAMESPACE"; + private const string K8sHostnameKey = "HOSTNAME"; + private const string K8sContainerNameKey = "KUBERNETES_CONTAINER_NAME"; + private const string K8sCertificatePath = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"; + private const string K8sCredentialPath = "/var/run/secrets/kubernetes.io/serviceaccount/token"; /// /// CGroup Parse Versions. @@ -33,6 +43,11 @@ internal enum ParseMode /// Represents CGroupV2. /// V2, + + /// + /// Represents Kubernetes. + /// + K8s, } /// @@ -41,24 +56,29 @@ internal enum ParseMode /// Resource with key-value pairs of resource attributes. public Resource Detect() { - var cGroupBuild = this.BuildResource(Filepath, ParseMode.V1); - if (cGroupBuild == Resource.Empty) + var resource = this.BuildResource(Filepath, ParseMode.K8s); + if (resource == Resource.Empty) { - cGroupBuild = this.BuildResource(FilepathV2, ParseMode.V2); + resource = this.BuildResource(Filepath, ParseMode.V1); } - return cGroupBuild; + if (resource == Resource.Empty) + { + resource = this.BuildResource(FilepathV2, ParseMode.V2); + } + + return resource; } /// /// Builds the resource attributes from Container Id in file path. /// /// File path where container id exists. - /// CGroup Version of file to parse from. + /// CGroup Version of file to parse from. /// Returns Resource with list of key-value pairs of container resource attributes if container id exists else empty resource. - internal Resource BuildResource(string path, ParseMode cgroupVersion) + internal Resource BuildResource(string path, ParseMode parseMode) { - var containerId = this.ExtractContainerId(path, cgroupVersion); + var containerId = this.ExtractContainerId(path, parseMode); if (string.IsNullOrEmpty(containerId)) { @@ -132,45 +152,125 @@ private static string RemovePrefixAndSuffixIfNeeded(string input, int startIndex return input.Substring(startIndex, endIndex - startIndex); } + private static string? ExtractK8sContainerId() + { + try + { + var host = Environment.GetEnvironmentVariable(K8sServiceHostKey); + var port = Environment.GetEnvironmentVariable(K8sServicePortKey); + var @namespace = Environment.GetEnvironmentVariable(K8sNamespaceKey); + var hostname = Environment.GetEnvironmentVariable(K8sHostnameKey); + var containerName = Environment.GetEnvironmentVariable(K8sContainerNameKey); + var url = $"https://{host}:{port}/api/v1/namespaces/{@namespace}/pods/{hostname}"; + var credentials = GetK8sCredentials(K8sCredentialPath); + using var httpClientHandler = ServerCertificateValidationHandler.Create(K8sCertificatePath, ContainerResourceEventSource.Log); + var response = ResourceDetectorUtils.SendOutRequest(url, "GET", new KeyValuePair("Authorization", credentials), httpClientHandler).GetAwaiter().GetResult(); + var pod = DeserializeK8sResponse(response); + + if (pod == null || pod.Status == null || pod.Status.ContainerStatuses == null) + { + return string.Empty; + } + + var container = pod.Status.ContainerStatuses.SingleOrDefault(p => p.Name == containerName); + if (container is null) + { + return string.Empty; + } + + // Container's ID is in :// format. + var index = container.Id.LastIndexOf('/'); + return container.Id.Substring(index + 1); + } + catch (Exception ex) + { + ContainerResourceEventSource.Log.ExtractResourceAttributesException($"{nameof(ContainerResourceDetector)}: Failed to extract container id", ex); + } + + return null; + + static string? GetK8sCredentials(string path) + { + try + { + var stringBuilder = new StringBuilder(); + + using (var streamReader = ResourceDetectorUtils.GetStreamReader(path)) + { + while (!streamReader.EndOfStream) + { + stringBuilder.Append(streamReader.ReadLine()?.Trim()); + } + } + + stringBuilder.Insert(0, "Bearer "); + + return stringBuilder.ToString(); + } + catch (Exception ex) + { + ContainerResourceEventSource.Log.ExtractResourceAttributesException($"{nameof(ContainerResourceDetector)}: Failed to load client token", ex); + } + + return null; + } + + static K8sPod? DeserializeK8sResponse(string response) + { +#if NET6_0_OR_GREATER + return ResourceDetectorUtils.DeserializeFromString(response, SourceGenerationContext.Default.K8sPod); +#else + return ResourceDetectorUtils.DeserializeFromString(response); +#endif + } + } + /// /// Extracts Container Id from path using the cgroupv1 format. /// /// cgroup path. - /// CGroup Version of file to parse from. - /// Container Id, Null if not found or exception being thrown. - private string? ExtractContainerId(string path, ParseMode cgroupVersion) + /// CGroup Version of file to parse from. + /// Container Id, if not found or exception being thrown. + private string? ExtractContainerId(string path, ParseMode parseMode) { try { - if (!File.Exists(path)) + if (parseMode == ParseMode.K8s) { - return null; + return ExtractK8sContainerId(); } - - foreach (string line in File.ReadLines(path)) + else { - string? containerId = null; - if (!string.IsNullOrEmpty(line)) + if (!File.Exists(path)) + { + return null; + } + + foreach (string line in File.ReadLines(path)) { - if (cgroupVersion == ParseMode.V1) + string? containerId = null; + if (!string.IsNullOrEmpty(line)) { - containerId = GetIdFromLineV1(line); + if (parseMode == ParseMode.V1) + { + containerId = GetIdFromLineV1(line); + } + else if (parseMode == ParseMode.V2 && line.Contains(Hostname, StringComparison.Ordinal)) + { + containerId = GetIdFromLineV2(line); + } } - else if (cgroupVersion == ParseMode.V2 && line.Contains(Hostname, StringComparison.Ordinal)) + + if (!string.IsNullOrEmpty(containerId)) { - containerId = GetIdFromLineV2(line); + return containerId; } } - - if (!string.IsNullOrEmpty(containerId)) - { - return containerId; - } } } catch (Exception ex) { - ContainerExtensionsEventSource.Log.ExtractResourceAttributesException($"{nameof(ContainerResourceDetector)} : Failed to extract Container id from path", ex); + ContainerResourceEventSource.Log.ExtractResourceAttributesException($"{nameof(ContainerResourceDetector)} : Failed to extract Container id from path", ex); } return null; diff --git a/src/OpenTelemetry.ResourceDetectors.Container/ContainerResourceEventSource.cs b/src/OpenTelemetry.ResourceDetectors.Container/ContainerResourceEventSource.cs new file mode 100644 index 0000000000..cffd2eb5ec --- /dev/null +++ b/src/OpenTelemetry.ResourceDetectors.Container/ContainerResourceEventSource.cs @@ -0,0 +1,59 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +using System; +using System.Diagnostics.Tracing; +using OpenTelemetry.Internal; + +namespace OpenTelemetry.ResourceDetectors.Container; + +[EventSource(Name = "OpenTelemetry-ResourceDetectors-Container")] +internal class ContainerResourceEventSource : EventSource, IServerCertificateValidationEventSource +{ + private const int EventIdFailedToExtractResourceAttributes = 1; + private const int EventIdFailedToValidateCertificate = 2; + private const int EventIdFailedToCreateHttpHandler = 3; + private const int EventIdFailedCertificateFileNotExists = 4; + private const int EventIdFailedToLoadCertificateInStorage = 5; + + public static ContainerResourceEventSource Log = new(); + + [NonEvent] + public void ExtractResourceAttributesException(string format, Exception ex) + { + if (this.IsEnabled(EventLevel.Error, (EventKeywords)(-1))) + { + this.FailedToExtractResourceAttributes(format, ex.ToInvariantString()); + } + } + + [Event(EventIdFailedToExtractResourceAttributes, Message = "Failed to extract resource attributes in '{0}'.", Level = EventLevel.Error)] + public void FailedToExtractResourceAttributes(string format, string exception) + { + this.WriteEvent(1, format, exception); + } + + [Event(EventIdFailedToValidateCertificate, Message = "Failed to validate certificate. Details: '{0}'", Level = EventLevel.Warning)] + public void FailedToValidateCertificate(string error) + { + this.WriteEvent(EventIdFailedToValidateCertificate, error); + } + + [Event(EventIdFailedToCreateHttpHandler, Message = "Failed to create HTTP handler. Exception: '{0}'", Level = EventLevel.Warning)] + public void FailedToCreateHttpHandler(Exception exception) + { + this.WriteEvent(EventIdFailedToCreateHttpHandler, exception.ToInvariantString()); + } + + [Event(EventIdFailedCertificateFileNotExists, Message = "Certificate file does not exist. File: '{0}'", Level = EventLevel.Warning)] + public void CertificateFileDoesNotExist(string filename) + { + this.WriteEvent(EventIdFailedCertificateFileNotExists, filename); + } + + [Event(EventIdFailedToLoadCertificateInStorage, Message = "Failed to load certificate in trusted storage. File: '{0}'", Level = EventLevel.Warning)] + public void FailedToLoadCertificateInTrustedStorage(string filename) + { + this.WriteEvent(EventIdFailedToLoadCertificateInStorage, filename); + } +} diff --git a/src/OpenTelemetry.ResourceDetectors.Container/Models/K8sContainerStatus.cs b/src/OpenTelemetry.ResourceDetectors.Container/Models/K8sContainerStatus.cs new file mode 100644 index 0000000000..eafe063be7 --- /dev/null +++ b/src/OpenTelemetry.ResourceDetectors.Container/Models/K8sContainerStatus.cs @@ -0,0 +1,15 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +using System.Text.Json.Serialization; + +namespace OpenTelemetry.ResourceDetectors.Container.Models; + +internal sealed class K8sContainerStatus +{ + [JsonPropertyName("name")] + public string Name { get; set; } = default!; + + [JsonPropertyName("containerID")] + public string Id { get; set; } = default!; +} diff --git a/src/OpenTelemetry.ResourceDetectors.Container/Models/K8sPod.cs b/src/OpenTelemetry.ResourceDetectors.Container/Models/K8sPod.cs new file mode 100644 index 0000000000..292f314cb7 --- /dev/null +++ b/src/OpenTelemetry.ResourceDetectors.Container/Models/K8sPod.cs @@ -0,0 +1,12 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +using System.Text.Json.Serialization; + +namespace OpenTelemetry.ResourceDetectors.Container.Models; + +internal sealed class K8sPod +{ + [JsonPropertyName("status")] + public K8sPodStatus? Status { get; set; } +} diff --git a/src/OpenTelemetry.ResourceDetectors.Container/Models/K8sPodStatus.cs b/src/OpenTelemetry.ResourceDetectors.Container/Models/K8sPodStatus.cs new file mode 100644 index 0000000000..f3afc06795 --- /dev/null +++ b/src/OpenTelemetry.ResourceDetectors.Container/Models/K8sPodStatus.cs @@ -0,0 +1,13 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace OpenTelemetry.ResourceDetectors.Container.Models; + +internal sealed class K8sPodStatus +{ + [JsonPropertyName("containerStatuses")] + public IReadOnlyList ContainerStatuses { get; set; } = new List(); +} diff --git a/src/OpenTelemetry.ResourceDetectors.Container/OpenTelemetry.ResourceDetectors.Container.csproj b/src/OpenTelemetry.ResourceDetectors.Container/OpenTelemetry.ResourceDetectors.Container.csproj index c77f1af5e6..8a57acfe65 100644 --- a/src/OpenTelemetry.ResourceDetectors.Container/OpenTelemetry.ResourceDetectors.Container.csproj +++ b/src/OpenTelemetry.ResourceDetectors.Container/OpenTelemetry.ResourceDetectors.Container.csproj @@ -11,5 +11,9 @@ + + + + diff --git a/src/OpenTelemetry.ResourceDetectors.Container/SourceGenerationContext.cs b/src/OpenTelemetry.ResourceDetectors.Container/SourceGenerationContext.cs new file mode 100644 index 0000000000..fc19df95b6 --- /dev/null +++ b/src/OpenTelemetry.ResourceDetectors.Container/SourceGenerationContext.cs @@ -0,0 +1,24 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +#if NET6_0_OR_GREATER +using System.Text.Json.Serialization; +using OpenTelemetry.ResourceDetectors.Container.Models; + +namespace OpenTelemetry.ResourceDetectors.Container; + +/// +/// "Source Generation" is feature added to System.Text.Json in .NET 6.0. +/// This is a performance optimization that avoids runtime reflection when performing serialization. +/// Serialization metadata will be computed at compile-time and included in the assembly. +/// . +/// . +/// . +/// +[JsonSerializable(typeof(K8sPod))] +[JsonSerializable(typeof(K8sPodStatus))] +[JsonSerializable(typeof(K8sContainerStatus))] +internal sealed partial class SourceGenerationContext : JsonSerializerContext +{ +} +#endif diff --git a/src/OpenTelemetry.ResourceDetectors.AWS/ResourceDetectorUtils.cs b/src/Shared/ResourceDetectorUtils.cs similarity index 98% rename from src/OpenTelemetry.ResourceDetectors.AWS/ResourceDetectorUtils.cs rename to src/Shared/ResourceDetectorUtils.cs index 331f8d86dd..d476e46f22 100644 --- a/src/OpenTelemetry.ResourceDetectors.AWS/ResourceDetectorUtils.cs +++ b/src/Shared/ResourceDetectorUtils.cs @@ -12,7 +12,7 @@ #endif using System.Threading.Tasks; -namespace OpenTelemetry.ResourceDetectors.AWS; +namespace OpenTelemetry.ResourceDetectors; /// /// Class for resource detector utils. diff --git a/test/OpenTelemetry.ResourceDetectors.Container.Tests/OpenTelemetry.ResourceDetectors.Container.Tests.csproj b/test/OpenTelemetry.ResourceDetectors.Container.Tests/OpenTelemetry.ResourceDetectors.Container.Tests.csproj index 6b11db9474..2bc9efb956 100644 --- a/test/OpenTelemetry.ResourceDetectors.Container.Tests/OpenTelemetry.ResourceDetectors.Container.Tests.csproj +++ b/test/OpenTelemetry.ResourceDetectors.Container.Tests/OpenTelemetry.ResourceDetectors.Container.Tests.csproj @@ -11,8 +11,4 @@ - - - -