diff --git a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/requests/LogsUploadRequest.cs b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/requests/LogsUploadRequest.cs new file mode 100644 index 00000000000..ef1cb957666 --- /dev/null +++ b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/requests/LogsUploadRequest.cs @@ -0,0 +1,29 @@ +// Copyright (c) Microsoft. All rights reserved. +namespace Microsoft.Azure.Devices.Edge.Agent.Core.Requests +{ + using Microsoft.Azure.Devices.Edge.Agent.Core.Logs; + using Microsoft.Azure.Devices.Edge.Util; + using Newtonsoft.Json; + + public class LogsUploadRequest + { + public LogsUploadRequest(string id, LogsContentEncoding encoding, LogsContentType contentType, string sasUrl) + { + this.Id = Preconditions.CheckNonWhiteSpace(id, nameof(id)); + this.Encoding = encoding; + this.ContentType = contentType; + this.SasUrl = sasUrl; + } + + [JsonConstructor] + LogsUploadRequest(string id, LogsContentEncoding? encoding, LogsContentType? contentType, string sasUrl) + : this(id, encoding.HasValue ? encoding.Value : LogsContentEncoding.None, contentType.HasValue ? contentType.Value : LogsContentType.Json, sasUrl) + { + } + + public string Id { get; } + public LogsContentEncoding Encoding { get; } + public LogsContentType ContentType { get; } + public string SasUrl { get; } + } +} diff --git a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/requests/LogsUploadRequestHandler.cs b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/requests/LogsUploadRequestHandler.cs new file mode 100644 index 00000000000..65ca017edf7 --- /dev/null +++ b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/requests/LogsUploadRequestHandler.cs @@ -0,0 +1,32 @@ +// Copyright (c) Microsoft. All rights reserved. +namespace Microsoft.Azure.Devices.Edge.Agent.Core.Requests +{ + using System; + using System.Threading; + using System.Threading.Tasks; + using Microsoft.Azure.Devices.Edge.Agent.Core.Logs; + using Microsoft.Azure.Devices.Edge.Util; + + public class LogsUploadRequestHandler : RequestHandlerBase + { + readonly ILogsUploader logsUploader; + readonly ILogsProvider logsProvider; + + public LogsUploadRequestHandler(ILogsUploader logsUploader, ILogsProvider logsProvider) + { + this.logsProvider = Preconditions.CheckNotNull(logsProvider, nameof(logsProvider)); + this.logsUploader = Preconditions.CheckNotNull(logsUploader, nameof(logsUploader)); + } + + public override string RequestName => "UploadLogs"; + + protected override async Task> HandleRequestInternal(Option payloadOption) + { + LogsUploadRequest payload = payloadOption.Expect(() => new ArgumentException("Request payload not found")); + var moduleLogOptions = new ModuleLogOptions(payload.Id, payload.Encoding, payload.ContentType); + byte[] logBytes = await this.logsProvider.GetLogs(moduleLogOptions, CancellationToken.None); + await this.logsUploader.Upload(payload.SasUrl, payload.Id, logBytes, payload.Encoding, payload.ContentType); + return Option.None(); + } + } +} diff --git a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.IoTHub/blob/AzureBlobLogsUploader.cs b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.IoTHub/blob/AzureBlobLogsUploader.cs index 90c6f8be7ff..0e07170ee1e 100644 --- a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.IoTHub/blob/AzureBlobLogsUploader.cs +++ b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.IoTHub/blob/AzureBlobLogsUploader.cs @@ -24,6 +24,11 @@ public class AzureBlobLogsUploader : ILogsUploader readonly string deviceId; readonly IAzureBlobUploader azureBlobUploader; + public AzureBlobLogsUploader(string iotHubName, string deviceId) + : this(iotHubName, deviceId, new AzureBlobUploader()) + { + } + public AzureBlobLogsUploader(string iotHubName, string deviceId, IAzureBlobUploader azureBlobUploader) { this.iotHubName = Preconditions.CheckNonWhiteSpace(iotHubName, nameof(iotHubName)); diff --git a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Service/Program.cs b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Service/Program.cs index ace03c16dfe..a7605202134 100644 --- a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Service/Program.cs +++ b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Service/Program.cs @@ -102,11 +102,16 @@ public static async Task MainAsync(IConfiguration configuration) Option productInfo = versionInfo != VersionInfo.Empty ? Option.Some(versionInfo.ToString()) : Option.None(); Option upstreamProtocol = configuration.GetValue(Constants.UpstreamProtocolKey).ToUpstreamProtocol(); Option proxy = Proxy.Parse(configuration.GetValue("https_proxy"), logger); + string iothubHostname = null; + string deviceId = null; switch (mode.ToLowerInvariant()) { case Constants.DockerMode: var dockerUri = new Uri(configuration.GetValue("DockerUri")); string deviceConnectionString = configuration.GetValue("DeviceConnectionString"); + IotHubConnectionStringBuilder connectionStringParser = IotHubConnectionStringBuilder.Create(deviceConnectionString); + deviceId = connectionStringParser.DeviceId; + iothubHostname = connectionStringParser.HostName; builder.RegisterModule(new AgentModule(maxRestartCount, intensiveCareTime, coolOffTimeUnitInSeconds, usePersistentStorage, storagePath)); builder.RegisterModule(new DockerModule(deviceConnectionString, edgeDeviceHostName, dockerUri, dockerAuthConfig, upstreamProtocol, proxy, productInfo)); break; @@ -114,8 +119,8 @@ public static async Task MainAsync(IConfiguration configuration) case Constants.IotedgedMode: string managementUri = configuration.GetValue(Constants.EdgeletManagementUriVariableName); string workloadUri = configuration.GetValue(Constants.EdgeletWorkloadUriVariableName); - string iothubHostname = configuration.GetValue(Constants.IotHubHostnameVariableName); - string deviceId = configuration.GetValue(Constants.DeviceIdVariableName); + iothubHostname = configuration.GetValue(Constants.IotHubHostnameVariableName); + deviceId = configuration.GetValue(Constants.DeviceIdVariableName); string moduleId = configuration.GetValue(Constants.ModuleIdVariableName, Constants.EdgeAgentModuleIdentityName); string moduleGenerationId = configuration.GetValue(Constants.EdgeletModuleGenerationIdVariableName); string apiVersion = configuration.GetValue(Constants.EdgeletApiVersionVariableName); @@ -130,7 +135,7 @@ public static async Task MainAsync(IConfiguration configuration) switch (configSourceConfig.ToLowerInvariant()) { case "twin": - builder.RegisterModule(new TwinConfigSourceModule(backupConfigFilePath, configuration, versionInfo, TimeSpan.FromSeconds(configRefreshFrequencySecs))); + builder.RegisterModule(new TwinConfigSourceModule(iothubHostname, deviceId, backupConfigFilePath, configuration, versionInfo, TimeSpan.FromSeconds(configRefreshFrequencySecs))); break; case "local": diff --git a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Service/modules/AgentModule.cs b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Service/modules/AgentModule.cs index e2d761faf61..a9712fe4dce 100644 --- a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Service/modules/AgentModule.cs +++ b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Service/modules/AgentModule.cs @@ -245,7 +245,7 @@ await c.Resolve>(), var environmentProvider = c.Resolve>(); var planner = c.Resolve>(); var planRunner = c.Resolve(); - var reporter = c.Resolve(); + var reporter = c.Resolve>(); var moduleIdentityLifecycleManager = c.Resolve(); var deploymentConfigInfoSerde = c.Resolve>(); var deploymentConfigInfoStore = c.Resolve>(); @@ -254,7 +254,7 @@ await c.Resolve>(), await configSource, await planner, planRunner, - reporter, + await reporter, moduleIdentityLifecycleManager, await environmentProvider, deploymentConfigInfoStore, diff --git a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Service/modules/EdgeletModule.cs b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Service/modules/EdgeletModule.cs index 1f541700ec8..30ec5b5e42d 100644 --- a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Service/modules/EdgeletModule.cs +++ b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Service/modules/EdgeletModule.cs @@ -101,9 +101,9 @@ protected override void Load(ContainerBuilder builder) .As>() .SingleInstance(); - // IModuleRuntimeInfoProvider - builder.Register(c => new RuntimeInfoProvider(c.Resolve())) - .As() + // Task + builder.Register(c => Task.FromResult(new RuntimeInfoProvider(c.Resolve()) as IRuntimeInfoProvider)) + .As>() .SingleInstance(); // Task diff --git a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Service/modules/FileConfigSourceModule.cs b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Service/modules/FileConfigSourceModule.cs index 8a28df897e6..79cec3da75c 100644 --- a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Service/modules/FileConfigSourceModule.cs +++ b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Service/modules/FileConfigSourceModule.cs @@ -42,8 +42,8 @@ protected override void Load(ContainerBuilder builder) // Task // TODO: When using a file backed config source we need to figure out // how reporting will work. - builder.Register(c => NullReporter.Instance as IReporter) - .As() + builder.Register(c => Task.FromResult(NullReporter.Instance as IReporter)) + .As>() .SingleInstance(); base.Load(builder); diff --git a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Service/modules/TwinConfigSourceModule.cs b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Service/modules/TwinConfigSourceModule.cs index 99080794e51..fd52be17c98 100644 --- a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Service/modules/TwinConfigSourceModule.cs +++ b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Service/modules/TwinConfigSourceModule.cs @@ -7,10 +7,12 @@ namespace Microsoft.Azure.Devices.Edge.Agent.Service.Modules using Autofac; using Microsoft.Azure.Devices.Edge.Agent.Core; using Microsoft.Azure.Devices.Edge.Agent.Core.ConfigSources; + using Microsoft.Azure.Devices.Edge.Agent.Core.Logs; using Microsoft.Azure.Devices.Edge.Agent.Core.Requests; using Microsoft.Azure.Devices.Edge.Agent.Core.Serde; using Microsoft.Azure.Devices.Edge.Agent.Docker; using Microsoft.Azure.Devices.Edge.Agent.IoTHub; + using Microsoft.Azure.Devices.Edge.Agent.IoTHub.Blob; using Microsoft.Azure.Devices.Edge.Agent.IoTHub.ConfigSources; using Microsoft.Azure.Devices.Edge.Agent.IoTHub.Reporters; using Microsoft.Azure.Devices.Edge.Util; @@ -23,13 +25,19 @@ public class TwinConfigSourceModule : Module readonly IConfiguration configuration; readonly VersionInfo versionInfo; readonly TimeSpan configRefreshFrequency; + readonly string deviceId; + readonly string iotHubHostName; public TwinConfigSourceModule( + string iotHubHostname, + string deviceId, string backupConfigFilePath, IConfiguration config, VersionInfo versionInfo, TimeSpan configRefreshFrequency) { + this.iotHubHostName = Preconditions.CheckNonWhiteSpace(iotHubHostname, nameof(iotHubHostname)); + this.deviceId = Preconditions.CheckNonWhiteSpace(deviceId, nameof(deviceId)); this.backupConfigFilePath = Preconditions.CheckNonWhiteSpace(backupConfigFilePath, nameof(backupConfigFilePath)); this.configuration = Preconditions.CheckNotNull(config, nameof(config)); this.versionInfo = Preconditions.CheckNotNull(versionInfo, nameof(versionInfo)); @@ -38,87 +46,110 @@ public TwinConfigSourceModule( protected override void Load(ContainerBuilder builder) { - // IRequestManager + // ILogsUploader + builder.Register(c => new AzureBlobLogsUploader(this.iotHubHostName, this.deviceId)) + .As() + .SingleInstance(); + + // Task builder.Register( - c => - { - var requestHandlers = new List - { - new PingRequestHandler() - }; - return new RequestManager(requestHandlers); - }) - .As() + async c => + { + var logsProcessor = new LogsProcessor(new LogMessageParser(this.iotHubHostName, this.deviceId)); + IRuntimeInfoProvider runtimeInfoProvider = await c.Resolve>(); + return new LogsProvider(runtimeInfoProvider, logsProcessor) as ILogsProvider; + }) + .As>() .SingleInstance(); - // IEdgeAgentConnection + // Task builder.Register( - c => + async c => + { + var logsUploader = c.Resolve(); + ILogsProvider logsProvider = await c.Resolve>(); + var requestHandlers = new List { - var requestManager = c.Resolve(); - var serde = c.Resolve>(); - var deviceClientprovider = c.Resolve(); - IEdgeAgentConnection edgeAgentConnection = new EdgeAgentConnection(deviceClientprovider, serde, requestManager, this.configRefreshFrequency); - return edgeAgentConnection; - }) - .As() + new PingRequestHandler(), + new LogsUploadRequestHandler(logsUploader, logsProvider) + }; + return new RequestManager(requestHandlers) as IRequestManager; + }) + .As>() + .SingleInstance(); + + // Task + builder.Register( + async c => + { + var serde = c.Resolve>(); + var deviceClientprovider = c.Resolve(); + IRequestManager requestManager = await c.Resolve>(); + IEdgeAgentConnection edgeAgentConnection = new EdgeAgentConnection(deviceClientprovider, serde, requestManager, this.configRefreshFrequency); + return edgeAgentConnection; + }) + .As>() .SingleInstance(); // Task builder.Register( - async c => - { - var serde = c.Resolve>(); - var edgeAgentConnection = c.Resolve(); - IEncryptionProvider encryptionProvider = await c.Resolve>(); - var twinConfigSource = new TwinConfigSource(edgeAgentConnection, this.configuration); - IConfigSource backupConfigSource = new FileBackupConfigSource(this.backupConfigFilePath, twinConfigSource, serde, encryptionProvider); - return backupConfigSource; - }) + async c => + { + var serde = c.Resolve>(); + var edgeAgentConnectionTask = c.Resolve>(); + IEncryptionProvider encryptionProvider = await c.Resolve>(); + IEdgeAgentConnection edgeAgentConnection = await edgeAgentConnectionTask; + var twinConfigSource = new TwinConfigSource(edgeAgentConnection, this.configuration); + IConfigSource backupConfigSource = new FileBackupConfigSource(this.backupConfigFilePath, twinConfigSource, serde, encryptionProvider); + return backupConfigSource; + }) .As>() .SingleInstance(); - // IReporter + // Task builder.Register( - c => + async c => + { + var runtimeInfoDeserializerTypes = new Dictionary + { + [DockerType] = typeof(DockerReportedRuntimeInfo), + [Constants.Unknown] = typeof(UnknownRuntimeInfo) + }; + + var edgeAgentDeserializerTypes = new Dictionary { - var runtimeInfoDeserializerTypes = new Dictionary - { - [DockerType] = typeof(DockerReportedRuntimeInfo), - [Constants.Unknown] = typeof(UnknownRuntimeInfo) - }; + [DockerType] = typeof(EdgeAgentDockerRuntimeModule), + [Constants.Unknown] = typeof(UnknownEdgeAgentModule) + }; - var edgeAgentDeserializerTypes = new Dictionary - { - [DockerType] = typeof(EdgeAgentDockerRuntimeModule), - [Constants.Unknown] = typeof(UnknownEdgeAgentModule) - }; + var edgeHubDeserializerTypes = new Dictionary + { + [DockerType] = typeof(EdgeHubDockerRuntimeModule), + [Constants.Unknown] = typeof(UnknownEdgeHubModule) + }; - var edgeHubDeserializerTypes = new Dictionary - { - [DockerType] = typeof(EdgeHubDockerRuntimeModule), - [Constants.Unknown] = typeof(UnknownEdgeHubModule) - }; + var moduleDeserializerTypes = new Dictionary + { + [DockerType] = typeof(DockerRuntimeModule) + }; - var moduleDeserializerTypes = new Dictionary - { - [DockerType] = typeof(DockerRuntimeModule) - }; + var deserializerTypesMap = new Dictionary> + { + { typeof(IRuntimeInfo), runtimeInfoDeserializerTypes }, + { typeof(IEdgeAgentModule), edgeAgentDeserializerTypes }, + { typeof(IEdgeHubModule), edgeHubDeserializerTypes }, + { typeof(IModule), moduleDeserializerTypes } + }; - var deserializerTypesMap = new Dictionary> - { - { typeof(IRuntimeInfo), runtimeInfoDeserializerTypes }, - { typeof(IEdgeAgentModule), edgeAgentDeserializerTypes }, - { typeof(IEdgeHubModule), edgeHubDeserializerTypes }, - { typeof(IModule), moduleDeserializerTypes } - }; + var edgeAgentConnectionTask = c.Resolve>(); + IEdgeAgentConnection edgeAgentConnection = await edgeAgentConnectionTask; - return new IoTHubReporter( - c.Resolve(), - new TypeSpecificSerDe(deserializerTypesMap), - this.versionInfo) as IReporter; - }) - .As() + return new IoTHubReporter( + edgeAgentConnection, + new TypeSpecificSerDe(deserializerTypesMap), + this.versionInfo) as IReporter; + }) + .As>() .SingleInstance(); base.Load(builder); diff --git a/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Core.Test/requests/LogsUploadRequestHandlerTest.cs b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Core.Test/requests/LogsUploadRequestHandlerTest.cs new file mode 100644 index 00000000000..eac9205593b --- /dev/null +++ b/edge-agent/test/Microsoft.Azure.Devices.Edge.Agent.Core.Test/requests/LogsUploadRequestHandlerTest.cs @@ -0,0 +1,54 @@ +// Copyright (c) Microsoft. All rights reserved. +namespace Microsoft.Azure.Devices.Edge.Agent.Core.Test.Requests +{ + using System; + using System.Collections.Generic; + using System.Threading; + using System.Threading.Tasks; + using Microsoft.Azure.Devices.Edge.Agent.Core.Logs; + using Microsoft.Azure.Devices.Edge.Agent.Core.Requests; + using Microsoft.Azure.Devices.Edge.Util; + using Microsoft.Azure.Devices.Edge.Util.Test.Common; + using Moq; + using Xunit; + + [Unit] + public class LogsUploadRequestHandlerTest + { + [Theory] + [MemberData(nameof(GetLogsUploadRequestHandlerData))] + public async Task TestLogsUploadRequest(string payload, string id, string sasUrl, LogsContentEncoding contentEncoding, LogsContentType contentType) + { + // Arrange + var logsUploader = new Mock(); + var logsProvider = new Mock(); + var uploadBytes = new byte[100]; + logsProvider.Setup(l => l.GetLogs(It.IsAny(), It.IsAny())) + .ReturnsAsync(uploadBytes); + logsUploader.Setup(l => l.Upload(sasUrl, id, uploadBytes, contentEncoding, contentType)) + .Returns(Task.CompletedTask); + + // Act + var logsUploadRequestHandler = new LogsUploadRequestHandler(logsUploader.Object, logsProvider.Object); + Option response = await logsUploadRequestHandler.HandleRequest(Option.Maybe(payload)); + + // Assert + Assert.False(response.HasValue); + } + + public static IEnumerable GetLogsUploadRequestHandlerData() + { + string sasUrl = $"https://test1.blob.core.windows.net/cont2?st={Guid.NewGuid()}"; + yield return new object[] { @"{""id"": ""edgeAgent"", ""sasUrl"": """"}".Replace("", sasUrl), "edgeAgent", sasUrl, LogsContentEncoding.None, LogsContentType.Json }; + + sasUrl = $"https://test1.blob.core.windows.net/cont2?st={Guid.NewGuid()}"; + yield return new object[] { @"{""id"": ""edgeAgent"", ""sasUrl"": """", ""encoding"": ""gzip""}".Replace("", sasUrl), "edgeAgent", sasUrl, LogsContentEncoding.Gzip, LogsContentType.Json }; + + sasUrl = $"https://test1.blob.core.windows.net/cont2?st={Guid.NewGuid()}"; + yield return new object[] { @"{""id"": ""edgeAgent"", ""sasUrl"": """", ""encoding"": ""gzip"", ""contentType"": ""text""}".Replace("", sasUrl), "mod1", sasUrl, LogsContentEncoding.Gzip, LogsContentType.Text }; + + sasUrl = $"https://test1.blob.core.windows.net/cont2?st={Guid.NewGuid()}"; + yield return new object[] { @"{""id"": ""edgeAgent"", ""sasUrl"": """", ""encoding"": ""none"", ""contentType"": ""json""}".Replace("", sasUrl), "edgeHub", sasUrl, LogsContentEncoding.None, LogsContentType.Json }; + } + } +}