Skip to content

Commit

Permalink
Add support to Quickstart for testing devices behind an HTTP proxy. (#…
Browse files Browse the repository at this point in the history 
…446)

Add support to Quickstart for testing devices behind an HTTP proxy.

This change adds two new CLI parameters `--proxy` and `--upstream-protocol`.
They correspond to `agent.env.https_proxy` and `agent.env.UpstreamProtocol`
in config.yaml.

`--proxy` defaults to the `https_proxy` environment variable on the
`Quickstart` process itself, if it's set.

`--upstream-protocol` is also used to override the transport used by
IoT SDK's `ServiceClient` and EventHub SDK's `EventHubClient` to
AMQP-over-websockets if necessary.

Also fixed JSON errors in the smoke test deployment templates that caused `jq`
to choke.
  • Loading branch information
@arsing
arsing authored Oct 17, 2018
1 parent 2a6f93e commit 1217b83
Show file tree
Hide file tree
Showing 9 changed files with 86 additions and 20 deletions.
2 changes: 1 addition & 1 deletion smoke/IotEdgeQuickstart/IotEdgeQuickstart.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
<ItemGroup>
<PackageReference Include="McMaster.Extensions.CommandLineUtils" Version="2.2.0-*" />
<PackageReference Include="Microsoft.Azure.Devices" Version="1.17.1" />
<PackageReference Include="Microsoft.Azure.EventHubs" Version="1.1.0" />
<PackageReference Include="Microsoft.Azure.EventHubs" Version="2.2.1" />
<PackageReference Include="Microsoft.CodeCoverage" Version="1.0.3" />
<PackageReference Include="RunProcessAsTask" Version="1.2.3" />
<PackageReference Include="YamlDotNet" Version="5.2.1" />
Expand Down
32 changes: 30 additions & 2 deletions smoke/IotEdgeQuickstart/Program.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ Option Environment variable
--registry registryAddress
--tag imageTag
--username registryUser
--proxy https_proxy
Defaults:
All options to this command have defaults. If an option is not specified and
Expand Down Expand Up @@ -55,6 +56,7 @@ Option Default value
--deployment deployment json file
--runtime-log-level debug
--clean_up_existing_device false
--proxy No proxy is used
"
)]
[HelpOption]
Expand Down Expand Up @@ -129,6 +131,12 @@ class Program
[Option("--clean_up_existing_device <true/false>", CommandOptionType.SingleValue, Description = "Clean up existing device on success.")]
public bool CleanUpExistingDeviceOnSuccess { get; } = false;

[Option("--proxy <value>", CommandOptionType.SingleValue, Description = "Proxy for IoT Hub connections.")]
public (bool useProxy, string proxyUrl) Proxy { get; } = (false, string.Empty);

[Option("--upstream-protocol <value>", CommandOptionType.SingleValue, Description = "Upstream protocol for IoT Hub connections.")]
public (bool overrideUpstreamProtocol, UpstreamProtocolType upstreamProtocol) UpstreamProtocol { get; } = (false, UpstreamProtocolType.Amqp);

// ReSharper disable once UnusedMember.Local
async Task<int> OnExecuteAsync()
{
Expand All @@ -152,17 +160,28 @@ async Task<int> OnExecuteAsync()
{
case BootstrapperType.Iotedged:
{
(bool useProxy, string proxyUrl) = this.Proxy;
Option<string> proxy = useProxy
? Option.Some(proxyUrl)
: Option.Maybe(Environment.GetEnvironmentVariable("https_proxy"));

if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
{
bootstrapper = new IotedgedWindows(this.BootstrapperArchivePath, credentials);
bootstrapper = new IotedgedWindows(this.BootstrapperArchivePath, credentials, proxy);
}
else
{
(bool useHttp, string hostname) = this.UseHttp;
Option<HttpUris> uris = useHttp
? Option.Some(string.IsNullOrEmpty(hostname) ? new HttpUris() : new HttpUris(hostname))
: Option.None<HttpUris>();
bootstrapper = new IotedgedLinux(this.BootstrapperArchivePath, credentials, uris);

(bool overrideUpstreamProtocol, UpstreamProtocolType upstreamProtocol) = this.UpstreamProtocol;
Option<UpstreamProtocolType> upstreamProtocolOption = overrideUpstreamProtocol
? Option.Some(upstreamProtocol)
: Option.None<UpstreamProtocolType>();

bootstrapper = new IotedgedLinux(this.BootstrapperArchivePath, credentials, uris, proxy, upstreamProtocolOption);
}
}
break;
Expand All @@ -188,6 +207,7 @@ async Task<int> OnExecuteAsync()
credentials,
connectionString,
endpoint,
this.UpstreamProtocol.Item2,
tag,
this.DeviceId,
this.EdgeHostname,
Expand Down Expand Up @@ -246,4 +266,12 @@ public enum LogLevel
Info,
Debug
}

public enum UpstreamProtocolType
{
Amqp,
AmqpWs,
Mqtt,
MqttWs
}
}
3 changes: 2 additions & 1 deletion smoke/IotEdgeQuickstart/Quickstart.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ public Quickstart(
Option<RegistryCredentials> credentials,
string iothubConnectionString,
string eventhubCompatibleEndpointWithEntityPath,
UpstreamProtocolType upstreamProtocol,
string imageTag,
string deviceId,
string hostname,
Expand All @@ -34,7 +35,7 @@ public Quickstart(
bool optimizedForPerformance,
LogLevel runtimeLogLevel,
bool cleanUpExistingDeviceOnSuccess) :
base(bootstrapper, credentials, iothubConnectionString, eventhubCompatibleEndpointWithEntityPath, imageTag, deviceId, hostname, deploymentFileName, deviceCaCert, deviceCaPk, deviceCaCerts, optimizedForPerformance, runtimeLogLevel, cleanUpExistingDeviceOnSuccess)
base(bootstrapper, credentials, iothubConnectionString, eventhubCompatibleEndpointWithEntityPath, upstreamProtocol, imageTag, deviceId, hostname, deploymentFileName, deviceCaCert, deviceCaPk, deviceCaCerts, optimizedForPerformance, runtimeLogLevel, cleanUpExistingDeviceOnSuccess)
{
this.leaveRunning = leaveRunning;
this.noDeployment = noDeployment;
Expand Down
33 changes: 29 additions & 4 deletions smoke/IotEdgeQuickstart/details/Details.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,13 +15,17 @@ namespace IotEdgeQuickstart.Details
using Microsoft.Azure.EventHubs;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using EventHubClientTransportType = Microsoft.Azure.EventHubs.TransportType;
using ServiceClientTransportType = Microsoft.Azure.Devices.TransportType;

public class Details
{
readonly IBootstrapper bootstrapper;
readonly Option<RegistryCredentials> credentials;
readonly string iothubConnectionString;
readonly string eventhubCompatibleEndpointWithEntityPath;
readonly ServiceClientTransportType serviceClientTransportType;
readonly EventHubClientTransportType eventHubClientTransportType;
readonly string imageTag;
readonly string deviceId;
readonly string hostname;
Expand All @@ -40,6 +44,7 @@ protected Details(
Option<RegistryCredentials> credentials,
string iothubConnectionString,
string eventhubCompatibleEndpointWithEntityPath,
UpstreamProtocolType upstreamProtocol,
string imageTag,
string deviceId,
string hostname,
Expand All @@ -56,6 +61,25 @@ bool cleanUpExistingDeviceOnSuccess
this.credentials = credentials;
this.iothubConnectionString = iothubConnectionString;
this.eventhubCompatibleEndpointWithEntityPath = eventhubCompatibleEndpointWithEntityPath;

switch (upstreamProtocol)
{
case UpstreamProtocolType.Amqp:
case UpstreamProtocolType.Mqtt:
this.serviceClientTransportType = ServiceClientTransportType.Amqp;
this.eventHubClientTransportType = EventHubClientTransportType.Amqp;
break;

case UpstreamProtocolType.AmqpWs:
case UpstreamProtocolType.MqttWs:
this.serviceClientTransportType = ServiceClientTransportType.Amqp_WebSocket_Only;
this.eventHubClientTransportType = EventHubClientTransportType.AmqpWebSockets;
break;

default:
throw new Exception($"Unexpected upstream protocol type {upstreamProtocol}");
}

this.imageTag = imageTag;
this.deviceId = deviceId;
this.hostname = hostname;
Expand Down Expand Up @@ -147,7 +171,7 @@ protected async Task VerifyEdgeAgentIsConnectedToIotHub()
try
{
ServiceClient serviceClient =
ServiceClient.CreateFromConnectionString(this.context.IotHubConnectionString);
ServiceClient.CreateFromConnectionString(this.context.IotHubConnectionString, this.serviceClientTransportType);

while (true)
{
Expand Down Expand Up @@ -196,6 +220,7 @@ protected Task DeployToEdgeDevice()
protected async Task VerifyDataOnIoTHub(string moduleId)
{
var builder = new EventHubsConnectionStringBuilder(this.eventhubCompatibleEndpointWithEntityPath);
builder.TransportType = this.eventHubClientTransportType;

Console.WriteLine($"Receiving events from device '{this.context.Device.Id}' on Event Hub '{builder.EntityPath}'");

Expand All @@ -207,7 +232,7 @@ protected async Task VerifyDataOnIoTHub(string moduleId)
EventHubPartitionKeyResolver.ResolveToPartition(
this.context.Device.Id,
(await eventHubClient.GetRuntimeInformationAsync()).PartitionCount),
DateTime.Now);
EventPosition.FromEnd());

var result = new TaskCompletionSource<bool>();
using (var cts = new CancellationTokenSource(TimeSpan.FromMinutes(3)))
Expand All @@ -216,8 +241,8 @@ protected async Task VerifyDataOnIoTHub(string moduleId)
{
eventHubReceiver.SetReceiveHandler(new PartitionReceiveHandler(eventData =>
{
eventData.Properties.TryGetValue("iothub-connection-device-id", out object devId);
eventData.Properties.TryGetValue("iothub-connection-module-id", out object modId);
eventData.SystemProperties.TryGetValue("iothub-connection-device-id", out object devId);
eventData.SystemProperties.TryGetValue("iothub-connection-module-id", out object modId);

if (devId != null && devId.ToString().Equals(this.context.Device.Id) &&
modId != null && modId.ToString().Equals(moduleId))
Expand Down
10 changes: 9 additions & 1 deletion smoke/IotEdgeQuickstart/details/IotedgedLinux.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,12 +64,16 @@ class IotedgedLinux : IBootstrapper
readonly string archivePath;
readonly Option<RegistryCredentials> credentials;
readonly Option<HttpUris> httpUris;
readonly Option<string> proxy;
readonly Option<UpstreamProtocolType> upstreamProtocol;

public IotedgedLinux(string archivePath, Option<RegistryCredentials> credentials, Option<HttpUris> httpUris)
public IotedgedLinux(string archivePath, Option<RegistryCredentials> credentials, Option<HttpUris> httpUris, Option<string> proxy, Option<UpstreamProtocolType> upstreamProtocol)
{
this.archivePath = archivePath;
this.credentials = credentials;
this.httpUris = httpUris;
this.proxy = proxy;
this.upstreamProtocol = upstreamProtocol;
}

public async Task VerifyNotActive()
Expand Down Expand Up @@ -198,6 +202,10 @@ public async Task Configure(string connectionString, string image, string hostna
doc.ReplaceOrAdd("certificates.trusted_ca_certs", deviceCaCerts);
}

this.proxy.ForEach(proxy => doc.ReplaceOrAdd("agent.env.https_proxy", proxy));

this.upstreamProtocol.ForEach(upstreamProtocol => doc.ReplaceOrAdd("agent.env.UpstreamProtocol", upstreamProtocol.ToString()));

string result = doc.ToString();


Expand Down
8 changes: 7 additions & 1 deletion smoke/IotEdgeQuickstart/details/IotedgedWindows.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,14 @@ class IotedgedWindows : IBootstrapper
{
readonly string archivePath;
readonly Option<RegistryCredentials> credentials;
readonly Option<string> proxy;
string scriptDir;

public IotedgedWindows(string archivePath, Option<RegistryCredentials> credentials)
public IotedgedWindows(string archivePath, Option<RegistryCredentials> credentials, Option<string> proxy)
{
this.archivePath = archivePath;
this.credentials = credentials;
this.proxy = proxy;
}

public async Task VerifyNotActive()
Expand Down Expand Up @@ -115,6 +117,10 @@ await Process.RunAsync("powershell",
args += $" -Username '{c.User}' -Password (ConvertTo-SecureString '{c.Password}' -AsPlainText -Force)";
}

this.proxy.ForEach(proxy => {
args += $" -Proxy '{proxy}'";
});

if (this.archivePath != null)
{
args += $" -ArchivePath '{this.archivePath}'";
Expand Down
2 changes: 1 addition & 1 deletion smoke/IotEdgeQuickstart/details/PartitionReceiveHandler.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,6 @@ public Task ProcessEventsAsync(IEnumerable<EventData> events)
return Task.CompletedTask;
}
public Task ProcessErrorAsync(Exception error) => throw error;
public int MaxBatchSize { get; } = 10;
public int MaxBatchSize { get; set; }
}
}
3 changes: 1 addition & 2 deletions smoke/IotEdgeQuickstart/e2e_deployment_files/dm_module_to_module_deployment.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,8 +81,7 @@
"image": "edgebuilds.azurecr.io/microsoft/azureiotedge-direct-method-receiver:<Build.BuildNumber>-linux-<Architecture>",
"createOptions": ""
}
},

}
}
}
},
Expand Down
13 changes: 6 additions & 7 deletions smoke/IotEdgeQuickstart/e2e_deployment_files/module_to_module_deployment.template.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,11 +31,11 @@
"image": "edgebuilds.azurecr.io/microsoft/azureiotedge-hub:<Build.BuildNumber>-linux-<Architecture>",
"createOptions": "{\"HostConfig\": {\"PortBindings\": {\"8883/tcp\": [{\"HostPort\": \"8883\"}],\"443/tcp\": [{\"HostPort\": \"443\"}],\"5671/tcp\": [{\"HostPort\": \"5671\"}]}}}"
},
"env": {
"OptimizeForPerformance": {
"value": "<OptimizeForPerformance>"
}
},
"env": {
"OptimizeForPerformance": {
"value": "<OptimizeForPerformance>"
}
},
"status": "running",
"restartPolicy": "always"
}
Expand Down Expand Up @@ -65,8 +65,7 @@
"image": "edgebuilds.azurecr.io/microsoft/azureiotedge-temperature-filter:<Build.BuildNumber>-linux-<Architecture>",
"createOptions": ""
}
},

}
}
}
},
Expand Down

0 comments on commit 1217b83

Please sign in to comment.