Skip to content

Commit

Permalink
[Perf, Stress] Case-insensitive command-line parsing (Azure#16965)
Browse files Browse the repository at this point in the history 
- Perf and Stress frameworks should use case-insensitive command-line parsing
- Refactor common code into PerfStressUtilities shared source
- Add PerfOptions.StatusInterval to align with StressOptions
- Fixes Azure#16850
  • Loading branch information
@mikeharder @annelo-msft
mikeharder authored and annelo-msft committed Feb 17, 2021
1 parent 5db638c commit 4d524b5
Show file tree
Hide file tree
Showing 9 changed files with 235 additions and 240 deletions.
4 changes: 4 additions & 0 deletions common/Perf/Azure.Test.Perf/Azure.Test.Perf.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,4 +8,8 @@
<Reference Include="System.Net.Http" Condition="'$(TargetFrameworkIdentifier)' == '.NETFramework'" />
</ItemGroup>

<ItemGroup>
<Compile Include="..\..\PerfStressShared\PerfStressUtilities.cs" Link="Shared\%(RecursiveDir)\%(Filename)%(Extension)" />
</ItemGroup>

</Project>
3 changes: 3 additions & 0 deletions common/Perf/Azure.Test.Perf/PerfOptions.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,9 @@ public class PerfOptions
[Option('r', "rate", HelpText = "Target throughput (ops/sec)")]
public int? Rate { get; set; }

[Option("status-interval", Default = 1, HelpText = "Interval to write status to console in seconds")]
public int StatusInterval { get; set; }

[Option("sync", HelpText = "Runs sync version of test")]
public bool Sync { get; set; }

Expand Down
137 changes: 17 additions & 120 deletions common/Perf/Azure.Test.Perf/PerfProgram.cs
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.

using Azure.Test.PerfStress;
using CommandLine;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Reflection;
using System.Reflection.Emit;
using System.Runtime;
using System.Text.Json;
using System.Threading;
Expand All @@ -31,12 +31,12 @@ public static async Task Main(Assembly assembly, string[] args)

if (testTypes.Any())
{
var optionTypes = GetOptionTypes(testTypes);
await Parser.Default.ParseArguments(args, optionTypes).MapResult<PerfOptions, Task>(
var optionTypes = PerfStressUtilities.GetOptionTypes(testTypes);
await PerfStressUtilities.Parser.ParseArguments(args, optionTypes).MapResult<PerfOptions, Task>(
async o =>
{
var verbName = o.GetType().GetCustomAttribute<VerbAttribute>().Name;
var testType = testTypes.Where(t => GetVerbName(t.Name) == verbName).Single();
var testType = testTypes.Where(t => PerfStressUtilities.GetVerbName(t.Name) == verbName).Single();
await Run(testType, o);
},
errors => Task.CompletedTask
Expand Down Expand Up @@ -83,7 +83,7 @@ private static async Task Run(Type testType, PerfOptions options)
Console.WriteLine();

using var setupStatusCts = new CancellationTokenSource();
var setupStatusThread = PrintStatus("=== Setup ===", () => ".", newLine: false, setupStatusCts.Token);
var setupStatusThread = PerfStressUtilities.PrintStatus("=== Setup ===", () => ".", newLine: false, setupStatusCts.Token);

using var cleanupStatusCts = new CancellationTokenSource();
Thread cleanupStatusThread = null;
Expand All @@ -109,7 +109,8 @@ private static async Task Run(Type testType, PerfOptions options)

if (options.Warmup > 0)
{
await RunTestsAsync(tests, options.Sync, options.Parallel, options.Rate, options.Warmup, "Warmup");
await RunTestsAsync(tests, options.Sync, options.Parallel, options.Rate, options.Warmup, options.StatusInterval,
"Warmup");
}

for (var i = 0; i < options.Iterations; i++)
Expand All @@ -119,7 +120,8 @@ private static async Task Run(Type testType, PerfOptions options)
{
title += " " + (i + 1);
}
await RunTestsAsync(tests, options.Sync, options.Parallel, options.Rate, options.Duration, title, options.JobStatistics, options.Latency);
await RunTestsAsync(tests, options.Sync, options.Parallel, options.Rate, options.Duration, options.StatusInterval,
title, options.JobStatistics, options.Latency);
}
}
finally
Expand All @@ -128,7 +130,7 @@ private static async Task Run(Type testType, PerfOptions options)
{
if (cleanupStatusThread == null)
{
cleanupStatusThread = PrintStatus("=== Cleanup ===", () => ".", newLine: false, cleanupStatusCts.Token);
cleanupStatusThread = PerfStressUtilities.PrintStatus("=== Cleanup ===", () => ".", newLine: false, cleanupStatusCts.Token);
}

await Task.WhenAll(tests.Select(t => t.CleanupAsync()));
Expand All @@ -141,7 +143,7 @@ private static async Task Run(Type testType, PerfOptions options)
{
if (cleanupStatusThread == null)
{
cleanupStatusThread = PrintStatus("=== Cleanup ===", () => ".", newLine: false, cleanupStatusCts.Token);
cleanupStatusThread = PerfStressUtilities.PrintStatus("=== Cleanup ===", () => ".", newLine: false, cleanupStatusCts.Token);
}

await tests[0].GlobalCleanupAsync();
Expand All @@ -161,7 +163,7 @@ private static async Task Run(Type testType, PerfOptions options)
}

private static async Task RunTestsAsync(IPerfTest[] tests, bool sync, int parallel, int? rate,
int durationSeconds, string title, bool jobStatistics = false, bool latency = false)
int durationSeconds, int statusIntervalSeconds, string title, bool jobStatistics = false, bool latency = false)
{
_completedOperations = new int[parallel];
_lastCompletionTimes = new TimeSpan[parallel];
Expand Down Expand Up @@ -191,7 +193,7 @@ private static async Task RunTestsAsync(IPerfTest[] tests, bool sync, int parall
var lastCompleted = 0;

using var progressStatusCts = new CancellationTokenSource();
var progressStatusThread = PrintStatus(
var progressStatusThread = PerfStressUtilities.PrintStatus(
$"=== {title} ===" + Environment.NewLine +
"Current\t\tTotal",
() =>
Expand All @@ -202,7 +204,9 @@ private static async Task RunTestsAsync(IPerfTest[] tests, bool sync, int parall
return currentCompleted + "\t\t" + totalCompleted;
},
newLine: true,
progressStatusCts.Token);
progressStatusCts.Token,
statusIntervalSeconds
);

Thread pendingOperationsThread = null;
if (rate.HasValue)
Expand Down Expand Up @@ -375,7 +379,7 @@ private static async Task RunLoopAsync(IPerfTest test, int index, bool latency,
catch (Exception e)
{
// Ignore if any part of the exception chain is type OperationCanceledException
if (!ContainsOperationCanceledException(e))
if (!PerfStressUtilities.ContainsOperationCanceledException(e))
{
throw;
}
Expand Down Expand Up @@ -406,112 +410,5 @@ private static Thread WritePendingOperations(int rate, CancellationToken token)

return thread;
}

// Run in dedicated thread instead of using async/await in ThreadPool, to ensure this thread has priority
// and never fails to run to due ThreadPool starvation.
private static Thread PrintStatus(string header, Func<object> status, bool newLine, CancellationToken token)
{
var thread = new Thread(() =>
{
Console.WriteLine(header);

bool needsExtraNewline = false;

while (!token.IsCancellationRequested)
{
try
{
Sleep(TimeSpan.FromSeconds(1), token);
}
catch (OperationCanceledException)
{
}

var obj = status();

if (newLine)
{
Console.WriteLine(obj);
}
else
{
Console.Write(obj);
needsExtraNewline = true;
}
}

if (needsExtraNewline)
{
Console.WriteLine();
}

Console.WriteLine();
});

thread.Start();

return thread;
}

private static void Sleep(TimeSpan timeout, CancellationToken token)
{
var sw = Stopwatch.StartNew();
while (sw.Elapsed < timeout)
{
if (token.IsCancellationRequested)
{
// Simulate behavior of Task.Delay(TimeSpan, CancellationToken)
throw new OperationCanceledException();
}

Thread.Sleep(TimeSpan.FromMilliseconds(10));
}
}

private static bool ContainsOperationCanceledException(Exception e)
{
if (e is OperationCanceledException)
{
return true;
}
else if (e.InnerException != null)
{
return ContainsOperationCanceledException(e.InnerException);
}
else
{
return false;
}
}

// Dynamically create option types with a "Verb" attribute
private static Type[] GetOptionTypes(IEnumerable<Type> testTypes)
{
var optionTypes = new List<Type>();

var ab = AssemblyBuilder.DefineDynamicAssembly(new AssemblyName("Options"), AssemblyBuilderAccess.Run);
var mb = ab.DefineDynamicModule("Options");

foreach (var t in testTypes)
{
var baseOptionsType = t.GetConstructors().First().GetParameters()[0].ParameterType;
var tb = mb.DefineType(t.Name + "Options", TypeAttributes.Public, baseOptionsType);

var attrCtor = typeof(VerbAttribute).GetConstructor(new Type[] { typeof(string), typeof(bool) });
var verbName = GetVerbName(t.Name);
tb.SetCustomAttribute(new CustomAttributeBuilder(attrCtor,
new object[] { verbName, attrCtor.GetParameters()[1].DefaultValue }));

optionTypes.Add(tb.CreateType());
}

return optionTypes.ToArray();
}

private static string GetVerbName(string testName)
{
var lower = testName.ToLowerInvariant();
return lower.EndsWith("test") ? lower.Substring(0, lower.Length - 4) : lower;
}
}
}
61 changes: 61 additions & 0 deletions common/PerfStressShared/Azure.Test.PerfStress.sln
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@

Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio Version 16
VisualStudioVersion = 16.0.29325.168
MinimumVisualStudioVersion = 10.0.40219.1
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Test.Stress", "..\Stress\Azure.Test.Stress\Azure.Test.Stress.csproj", "{A0CCD9EF-D7C7-409B-9D28-19E3C4BD1948}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Sample.Stress", "..\Stress\Azure.Sample.Stress\Azure.Sample.Stress.csproj", "{29C9D6CF-DE90-42FD-B58F-3C254334D2C7}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Template", "..\..\sdk\template\Azure.Template\src\Azure.Template.csproj", "{6E88344A-4557-424C-9467-9A2AFAD84134}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Template.Stress", "..\..\sdk\template\Azure.Template\stress\Azure.Template.Stress.csproj", "{5B4187EE-9432-4FAD-9266-57D02A6A05F2}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Test.Perf", "..\Perf\Azure.Test.Perf\Azure.Test.Perf.csproj", "{F4069F59-77C7-4A81-A5CB-D358906B6D5B}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Sample.Perf", "..\Perf\Azure.Sample.Perf\Azure.Sample.Perf.csproj", "{29C0605C-5C5D-44A5-8743-BC027D1B9C24}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Template.Perf", "..\..\sdk\template\Azure.Template\perf\Azure.Template.Perf.csproj", "{D4B44894-7439-4242-B94D-D97956EAD938}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Release|Any CPU = Release|Any CPU
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{A0CCD9EF-D7C7-409B-9D28-19E3C4BD1948}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{A0CCD9EF-D7C7-409B-9D28-19E3C4BD1948}.Debug|Any CPU.Build.0 = Debug|Any CPU
{A0CCD9EF-D7C7-409B-9D28-19E3C4BD1948}.Release|Any CPU.ActiveCfg = Release|Any CPU
{A0CCD9EF-D7C7-409B-9D28-19E3C4BD1948}.Release|Any CPU.Build.0 = Release|Any CPU
{29C9D6CF-DE90-42FD-B58F-3C254334D2C7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{29C9D6CF-DE90-42FD-B58F-3C254334D2C7}.Debug|Any CPU.Build.0 = Debug|Any CPU
{29C9D6CF-DE90-42FD-B58F-3C254334D2C7}.Release|Any CPU.ActiveCfg = Release|Any CPU
{29C9D6CF-DE90-42FD-B58F-3C254334D2C7}.Release|Any CPU.Build.0 = Release|Any CPU
{6E88344A-4557-424C-9467-9A2AFAD84134}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{6E88344A-4557-424C-9467-9A2AFAD84134}.Debug|Any CPU.Build.0 = Debug|Any CPU
{6E88344A-4557-424C-9467-9A2AFAD84134}.Release|Any CPU.ActiveCfg = Release|Any CPU
{6E88344A-4557-424C-9467-9A2AFAD84134}.Release|Any CPU.Build.0 = Release|Any CPU
{5B4187EE-9432-4FAD-9266-57D02A6A05F2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{5B4187EE-9432-4FAD-9266-57D02A6A05F2}.Debug|Any CPU.Build.0 = Debug|Any CPU
{5B4187EE-9432-4FAD-9266-57D02A6A05F2}.Release|Any CPU.ActiveCfg = Release|Any CPU
{5B4187EE-9432-4FAD-9266-57D02A6A05F2}.Release|Any CPU.Build.0 = Release|Any CPU
{F4069F59-77C7-4A81-A5CB-D358906B6D5B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{F4069F59-77C7-4A81-A5CB-D358906B6D5B}.Debug|Any CPU.Build.0 = Debug|Any CPU
{F4069F59-77C7-4A81-A5CB-D358906B6D5B}.Release|Any CPU.ActiveCfg = Release|Any CPU
{F4069F59-77C7-4A81-A5CB-D358906B6D5B}.Release|Any CPU.Build.0 = Release|Any CPU
{29C0605C-5C5D-44A5-8743-BC027D1B9C24}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{29C0605C-5C5D-44A5-8743-BC027D1B9C24}.Debug|Any CPU.Build.0 = Debug|Any CPU
{29C0605C-5C5D-44A5-8743-BC027D1B9C24}.Release|Any CPU.ActiveCfg = Release|Any CPU
{29C0605C-5C5D-44A5-8743-BC027D1B9C24}.Release|Any CPU.Build.0 = Release|Any CPU
{D4B44894-7439-4242-B94D-D97956EAD938}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{D4B44894-7439-4242-B94D-D97956EAD938}.Debug|Any CPU.Build.0 = Debug|Any CPU
{D4B44894-7439-4242-B94D-D97956EAD938}.Release|Any CPU.ActiveCfg = Release|Any CPU
{D4B44894-7439-4242-B94D-D97956EAD938}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {CD67C1F7-9BEF-4878-A675-898D5D3B48E3}
EndGlobalSection
EndGlobal
Loading

0 comments on commit 4d524b5

Please sign in to comment.