Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Perf, Stress] Case-insensitive command-line parsing #16965

Merged
merged 8 commits into from
Nov 17, 2020
Merged

[Perf, Stress] Case-insensitive command-line parsing #16965

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
6dffeab
Case-insensitive command-line parsing
mikeharder Nov 11, 2020
ab0a057
Use CamelCase test names
mikeharder Nov 11, 2020
244a3c2
Move common code into shared folder
mikeharder Nov 12, 2020
fe409ac
Rename _defaultParser to _parser
mikeharder Nov 13, 2020
d188049
Remove PerfStressSharedSources MSBuild variable
mikeharder Nov 14, 2020
e85844d
Update casing in template PR path include
mikeharder Nov 16, 2020
277341a
Revert "Update casing in template PR path include"
mikeharder Nov 16, 2020
e349fc1
Revert "Revert "Update casing in template PR path include""
mikeharder Nov 16, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions common/Perf/Azure.Test.Perf/Azure.Test.Perf.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,4 +8,8 @@
<Reference Include="System.Net.Http" Condition="'$(TargetFrameworkIdentifier)' == '.NETFramework'" />
</ItemGroup>

<ItemGroup>
<Compile Include="$(PerfStressSharedSources)PerfStressUtilities.cs" Link="Shared\%(RecursiveDir)\%(Filename)%(Extension)" />
</ItemGroup>

</Project>
3 changes: 3 additions & 0 deletions common/Perf/Azure.Test.Perf/PerfOptions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,9 @@ public class PerfOptions
[Option('r', "rate", HelpText = "Target throughput (ops/sec)")]
public int? Rate { get; set; }

[Option("status-interval", Default = 1, HelpText = "Interval to write status to console in seconds")]
public int StatusInterval { get; set; }

[Option("sync", HelpText = "Runs sync version of test")]
public bool Sync { get; set; }

Expand Down
137 changes: 17 additions & 120 deletions common/Perf/Azure.Test.Perf/PerfProgram.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.

using Azure.Test.PerfStress;
using CommandLine;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Reflection;
using System.Reflection.Emit;
using System.Runtime;
using System.Text.Json;
using System.Threading;
Expand All @@ -31,12 +31,12 @@ public static async Task Main(Assembly assembly, string[] args)

if (testTypes.Any())
{
var optionTypes = GetOptionTypes(testTypes);
await Parser.Default.ParseArguments(args, optionTypes).MapResult<PerfOptions, Task>(
var optionTypes = PerfStressUtilities.GetOptionTypes(testTypes);
await PerfStressUtilities.Parser.ParseArguments(args, optionTypes).MapResult<PerfOptions, Task>(
async o =>
{
var verbName = o.GetType().GetCustomAttribute<VerbAttribute>().Name;
var testType = testTypes.Where(t => GetVerbName(t.Name) == verbName).Single();
var testType = testTypes.Where(t => PerfStressUtilities.GetVerbName(t.Name) == verbName).Single();
await Run(testType, o);
},
errors => Task.CompletedTask
Expand Down Expand Up @@ -83,7 +83,7 @@ private static async Task Run(Type testType, PerfOptions options)
Console.WriteLine();

using var setupStatusCts = new CancellationTokenSource();
var setupStatusThread = PrintStatus("=== Setup ===", () => ".", newLine: false, setupStatusCts.Token);
var setupStatusThread = PerfStressUtilities.PrintStatus("=== Setup ===", () => ".", newLine: false, setupStatusCts.Token);

using var cleanupStatusCts = new CancellationTokenSource();
Thread cleanupStatusThread = null;
Expand All @@ -109,7 +109,8 @@ private static async Task Run(Type testType, PerfOptions options)

if (options.Warmup > 0)
{
await RunTestsAsync(tests, options.Sync, options.Parallel, options.Rate, options.Warmup, "Warmup");
await RunTestsAsync(tests, options.Sync, options.Parallel, options.Rate, options.Warmup, options.StatusInterval,
"Warmup");
}

for (var i = 0; i < options.Iterations; i++)
Expand All @@ -119,7 +120,8 @@ private static async Task Run(Type testType, PerfOptions options)
{
title += " " + (i + 1);
}
await RunTestsAsync(tests, options.Sync, options.Parallel, options.Rate, options.Duration, title, options.JobStatistics, options.Latency);
await RunTestsAsync(tests, options.Sync, options.Parallel, options.Rate, options.Duration, options.StatusInterval,
title, options.JobStatistics, options.Latency);
}
}
finally
Expand All @@ -128,7 +130,7 @@ private static async Task Run(Type testType, PerfOptions options)
{
if (cleanupStatusThread == null)
{
cleanupStatusThread = PrintStatus("=== Cleanup ===", () => ".", newLine: false, cleanupStatusCts.Token);
cleanupStatusThread = PerfStressUtilities.PrintStatus("=== Cleanup ===", () => ".", newLine: false, cleanupStatusCts.Token);
}

await Task.WhenAll(tests.Select(t => t.CleanupAsync()));
Expand All @@ -141,7 +143,7 @@ private static async Task Run(Type testType, PerfOptions options)
{
if (cleanupStatusThread == null)
{
cleanupStatusThread = PrintStatus("=== Cleanup ===", () => ".", newLine: false, cleanupStatusCts.Token);
cleanupStatusThread = PerfStressUtilities.PrintStatus("=== Cleanup ===", () => ".", newLine: false, cleanupStatusCts.Token);
}

await tests[0].GlobalCleanupAsync();
Expand All @@ -161,7 +163,7 @@ private static async Task Run(Type testType, PerfOptions options)
}

private static async Task RunTestsAsync(IPerfTest[] tests, bool sync, int parallel, int? rate,
int durationSeconds, string title, bool jobStatistics = false, bool latency = false)
int durationSeconds, int statusIntervalSeconds, string title, bool jobStatistics = false, bool latency = false)
{
_completedOperations = new int[parallel];
_lastCompletionTimes = new TimeSpan[parallel];
Expand Down Expand Up @@ -191,7 +193,7 @@ private static async Task RunTestsAsync(IPerfTest[] tests, bool sync, int parall
var lastCompleted = 0;

using var progressStatusCts = new CancellationTokenSource();
var progressStatusThread = PrintStatus(
var progressStatusThread = PerfStressUtilities.PrintStatus(
$"=== {title} ===" + Environment.NewLine +
"Current\t\tTotal",
() =>
Expand All @@ -202,7 +204,9 @@ private static async Task RunTestsAsync(IPerfTest[] tests, bool sync, int parall
return currentCompleted + "\t\t" + totalCompleted;
},
newLine: true,
progressStatusCts.Token);
progressStatusCts.Token,
statusIntervalSeconds
);

Thread pendingOperationsThread = null;
if (rate.HasValue)
Expand Down Expand Up @@ -375,7 +379,7 @@ private static async Task RunLoopAsync(IPerfTest test, int index, bool latency,
catch (Exception e)
{
// Ignore if any part of the exception chain is type OperationCanceledException
if (!ContainsOperationCanceledException(e))
if (!PerfStressUtilities.ContainsOperationCanceledException(e))
{
throw;
}
Expand Down Expand Up @@ -406,112 +410,5 @@ private static Thread WritePendingOperations(int rate, CancellationToken token)

return thread;
}

// Run in dedicated thread instead of using async/await in ThreadPool, to ensure this thread has priority
// and never fails to run to due ThreadPool starvation.
private static Thread PrintStatus(string header, Func<object> status, bool newLine, CancellationToken token)
{
var thread = new Thread(() =>
{
Console.WriteLine(header);

bool needsExtraNewline = false;

while (!token.IsCancellationRequested)
{
try
{
Sleep(TimeSpan.FromSeconds(1), token);
}
catch (OperationCanceledException)
{
}

var obj = status();

if (newLine)
{
Console.WriteLine(obj);
}
else
{
Console.Write(obj);
needsExtraNewline = true;
}
}

if (needsExtraNewline)
{
Console.WriteLine();
}

Console.WriteLine();
});

thread.Start();

return thread;
}

private static void Sleep(TimeSpan timeout, CancellationToken token)
{
var sw = Stopwatch.StartNew();
while (sw.Elapsed < timeout)
{
if (token.IsCancellationRequested)
{
// Simulate behavior of Task.Delay(TimeSpan, CancellationToken)
throw new OperationCanceledException();
}

Thread.Sleep(TimeSpan.FromMilliseconds(10));
}
}

private static bool ContainsOperationCanceledException(Exception e)
{
if (e is OperationCanceledException)
{
return true;
}
else if (e.InnerException != null)
{
return ContainsOperationCanceledException(e.InnerException);
}
else
{
return false;
}
}

// Dynamically create option types with a "Verb" attribute
private static Type[] GetOptionTypes(IEnumerable<Type> testTypes)
{
var optionTypes = new List<Type>();

var ab = AssemblyBuilder.DefineDynamicAssembly(new AssemblyName("Options"), AssemblyBuilderAccess.Run);
var mb = ab.DefineDynamicModule("Options");

foreach (var t in testTypes)
{
var baseOptionsType = t.GetConstructors().First().GetParameters()[0].ParameterType;
var tb = mb.DefineType(t.Name + "Options", TypeAttributes.Public, baseOptionsType);

var attrCtor = typeof(VerbAttribute).GetConstructor(new Type[] { typeof(string), typeof(bool) });
var verbName = GetVerbName(t.Name);
tb.SetCustomAttribute(new CustomAttributeBuilder(attrCtor,
new object[] { verbName, attrCtor.GetParameters()[1].DefaultValue }));

optionTypes.Add(tb.CreateType());
}

return optionTypes.ToArray();
}

private static string GetVerbName(string testName)
{
var lower = testName.ToLowerInvariant();
return lower.EndsWith("test") ? lower.Substring(0, lower.Length - 4) : lower;
}
}
}
61 changes: 61 additions & 0 deletions common/PerfStressShared/Azure.Test.PerfStress.sln
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@

Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio Version 16
VisualStudioVersion = 16.0.29325.168
MinimumVisualStudioVersion = 10.0.40219.1
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Test.Stress", "..\Stress\Azure.Test.Stress\Azure.Test.Stress.csproj", "{A0CCD9EF-D7C7-409B-9D28-19E3C4BD1948}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Sample.Stress", "..\Stress\Azure.Sample.Stress\Azure.Sample.Stress.csproj", "{29C9D6CF-DE90-42FD-B58F-3C254334D2C7}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Template", "..\..\sdk\template\Azure.Template\src\Azure.Template.csproj", "{6E88344A-4557-424C-9467-9A2AFAD84134}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Template.Stress", "..\..\sdk\template\Azure.Template\stress\Azure.Template.Stress.csproj", "{5B4187EE-9432-4FAD-9266-57D02A6A05F2}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Test.Perf", "..\Perf\Azure.Test.Perf\Azure.Test.Perf.csproj", "{F4069F59-77C7-4A81-A5CB-D358906B6D5B}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Sample.Perf", "..\Perf\Azure.Sample.Perf\Azure.Sample.Perf.csproj", "{29C0605C-5C5D-44A5-8743-BC027D1B9C24}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Azure.Template.Perf", "..\..\sdk\template\Azure.Template\perf\Azure.Template.Perf.csproj", "{D4B44894-7439-4242-B94D-D97956EAD938}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Release|Any CPU = Release|Any CPU
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{A0CCD9EF-D7C7-409B-9D28-19E3C4BD1948}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{A0CCD9EF-D7C7-409B-9D28-19E3C4BD1948}.Debug|Any CPU.Build.0 = Debug|Any CPU
{A0CCD9EF-D7C7-409B-9D28-19E3C4BD1948}.Release|Any CPU.ActiveCfg = Release|Any CPU
{A0CCD9EF-D7C7-409B-9D28-19E3C4BD1948}.Release|Any CPU.Build.0 = Release|Any CPU
{29C9D6CF-DE90-42FD-B58F-3C254334D2C7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{29C9D6CF-DE90-42FD-B58F-3C254334D2C7}.Debug|Any CPU.Build.0 = Debug|Any CPU
{29C9D6CF-DE90-42FD-B58F-3C254334D2C7}.Release|Any CPU.ActiveCfg = Release|Any CPU
{29C9D6CF-DE90-42FD-B58F-3C254334D2C7}.Release|Any CPU.Build.0 = Release|Any CPU
{6E88344A-4557-424C-9467-9A2AFAD84134}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{6E88344A-4557-424C-9467-9A2AFAD84134}.Debug|Any CPU.Build.0 = Debug|Any CPU
{6E88344A-4557-424C-9467-9A2AFAD84134}.Release|Any CPU.ActiveCfg = Release|Any CPU
{6E88344A-4557-424C-9467-9A2AFAD84134}.Release|Any CPU.Build.0 = Release|Any CPU
{5B4187EE-9432-4FAD-9266-57D02A6A05F2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{5B4187EE-9432-4FAD-9266-57D02A6A05F2}.Debug|Any CPU.Build.0 = Debug|Any CPU
{5B4187EE-9432-4FAD-9266-57D02A6A05F2}.Release|Any CPU.ActiveCfg = Release|Any CPU
{5B4187EE-9432-4FAD-9266-57D02A6A05F2}.Release|Any CPU.Build.0 = Release|Any CPU
{F4069F59-77C7-4A81-A5CB-D358906B6D5B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{F4069F59-77C7-4A81-A5CB-D358906B6D5B}.Debug|Any CPU.Build.0 = Debug|Any CPU
{F4069F59-77C7-4A81-A5CB-D358906B6D5B}.Release|Any CPU.ActiveCfg = Release|Any CPU
{F4069F59-77C7-4A81-A5CB-D358906B6D5B}.Release|Any CPU.Build.0 = Release|Any CPU
{29C0605C-5C5D-44A5-8743-BC027D1B9C24}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{29C0605C-5C5D-44A5-8743-BC027D1B9C24}.Debug|Any CPU.Build.0 = Debug|Any CPU
{29C0605C-5C5D-44A5-8743-BC027D1B9C24}.Release|Any CPU.ActiveCfg = Release|Any CPU
{29C0605C-5C5D-44A5-8743-BC027D1B9C24}.Release|Any CPU.Build.0 = Release|Any CPU
{D4B44894-7439-4242-B94D-D97956EAD938}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{D4B44894-7439-4242-B94D-D97956EAD938}.Debug|Any CPU.Build.0 = Debug|Any CPU
{D4B44894-7439-4242-B94D-D97956EAD938}.Release|Any CPU.ActiveCfg = Release|Any CPU
{D4B44894-7439-4242-B94D-D97956EAD938}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {CD67C1F7-9BEF-4878-A675-898D5D3B48E3}
EndGlobalSection
EndGlobal
Loading