don't push

adamsitnik · Nov 17, 2020 · d02c372 · d02c372
1 parent 021c7be
commit d02c372
Show file tree

Hide file tree

Showing 5 changed files with 26 additions and 19 deletions.
diff --git a/src/benchmarks/micro/MicroBenchmarks.csproj b/src/benchmarks/micro/MicroBenchmarks.csproj
@@ -4,8 +4,8 @@
     <!-- Used by Python script to narrow down the specified target frameworks to test, and avoid downloading all supported SDKs -->
     <TargetFrameworks>$(PERFLAB_TARGET_FRAMEWORKS)</TargetFrameworks>
     <!-- Supported target frameworks -->
-    <TargetFrameworks Condition="'$(TargetFrameworks)' == '' AND '$(OS)' == 'Windows_NT'">net461;netcoreapp2.1;netcoreapp3.1;netcoreapp5.0;net6.0</TargetFrameworks>
-    <TargetFrameworks Condition="'$(TargetFrameworks)' == ''">netcoreapp2.1;netcoreapp3.1;netcoreapp5.0;net6.0</TargetFrameworks>
+    <TargetFrameworks Condition="'$(TargetFrameworks)' == '' AND '$(OS)' == 'Windows_NT'">net461;netcoreapp2.1;netcoreapp3.1;netcoreapp5.0</TargetFrameworks>
+    <TargetFrameworks Condition="'$(TargetFrameworks)' == ''">netcoreapp2.1;netcoreapp3.1;netcoreapp5.0</TargetFrameworks>
     <NoWarn>$(NoWarn);CS8002</NoWarn>
     <!-- Suppress binaryformatter obsolete warning -->
     <NoWarn>$(NoWarn);SYSLIB0011</NoWarn>

diff --git a/src/benchmarks/micro/libraries/System.Net.Primitives/DnsTests.cs b/src/benchmarks/micro/libraries/System.Net.Primitives/DnsTests.cs
@@ -10,7 +10,7 @@ namespace System.Net.Tests
     [BenchmarkCategory(Categories.Libraries, Categories.NoWASM)]
     public class DnsTests
     {
-        [Benchmark]
+        //[Benchmark]
         public IPHostEntry GetHostEntry() => Dns.GetHostEntry("34.206.253.53");
 
         [Benchmark]

diff --git a/src/harness/BenchmarkDotNet.Extensions/RecommendedConfig.cs b/src/harness/BenchmarkDotNet.Extensions/RecommendedConfig.cs
@@ -24,21 +24,27 @@ public static IConfig Create(
             Job job = null,
             bool getDiffableDisasm = false)
         {
-            if (job is null)
-            {
-                job = Job.Default
+                Job job1 = Job.Default
                     .WithWarmupCount(1) // 1 warmup is enough for our purpose
                     .WithIterationTime(TimeInterval.FromMilliseconds(250)) // the default is 0.5s per iteration, which is slighlty too much for us
-                    .WithMinIterationCount(15)
-                    .WithMaxIterationCount(20) // we don't want to run more that 20 iterations
-                    .DontEnforcePowerPlan(); // make sure BDN does not try to enforce High Performance power plan on Windows
+                    .WithMinIterationCount(35)
+                    .WithMaxIterationCount(40) // we don't want to run more that 20 iterations
+                    .WithOutlierMode(Perfolizer.Mathematics.OutlierDetection.OutlierMode.RemoveUpper)
+                    .WithMemoryRandomization(false)
+                    .DontEnforcePowerPlan()
+                    .AsBaseline(); // make sure BDN does not try to enforce High Performance power plan on Windows
 
-                // See https://github.com/dotnet/roslyn/issues/42393
-                job = job.WithArguments(new Argument[] { new MsBuildArgument("/p:DebugType=portable") });
-            }
+                Job job2 = Job.Default
+                    .WithWarmupCount(1) // 1 warmup is enough for our purpose
+                    .WithIterationTime(TimeInterval.FromMilliseconds(250)) // the default is 0.5s per iteration, which is slighlty too much for us
+                    .WithMinIterationCount(35)
+                    .WithMaxIterationCount(40) // we don't want to run more that 20 iterations
+                    .WithOutlierMode(Perfolizer.Mathematics.OutlierDetection.OutlierMode.DontRemove)
+                    .WithMemoryRandomization(true)
+                    .DontEnforcePowerPlan(); // make sure BDN does not try to enforce High Performance power plan on Windows
 
             var config = DefaultConfig.Instance
-                .AddJob(job.AsDefault()) // tell BDN that this are our default settings
+                .AddJob(job1).AddJob(job2) // tell BDN that this are our default settings
                 .WithArtifactsPath(artifactsPath.FullName)
                 .AddDiagnoser(MemoryDiagnoser.Default) // MemoryDiagnoser is enabled by default
                 .AddFilter(new OperatingSystemFilter())

diff --git a/src/tools/ResultsComparer/DataTransferContracts.cs b/src/tools/ResultsComparer/DataTransferContracts.cs
@@ -76,8 +76,8 @@ public class Statistics
         public double StandardError { get; set; }
         public double Variance { get; set; }
         public double StandardDeviation { get; set; }
-        public double Skewness { get; set; }
-        public double Kurtosis { get; set; }
+        public double? Skewness { get; set; }
+        public double? Kurtosis { get; set; }
         public ConfidenceInterval ConfidenceInterval { get; set; }
         public Percentiles Percentiles { get; set; }
     }

diff --git a/src/tools/ResultsComparer/Program.cs b/src/tools/ResultsComparer/Program.cs
@@ -115,16 +115,17 @@ private static void PrintSummary((string id, Benchmark baseResult, Benchmark dif
         private static void PrintTable((string id, Benchmark baseResult, Benchmark diffResult, EquivalenceTestConclusion conclusion)[] notSame, EquivalenceTestConclusion conclusion, CommandLineOptions args)
         {
             var data = notSame
-                .Where(result => result.conclusion == conclusion)
+                .Where(result => result.conclusion == conclusion && GetModalInfo(result.diffResult) != null)
                 .OrderByDescending(result => GetRatio(conclusion, result.baseResult, result.diffResult))
                 .Take(args.TopCount ?? int.MaxValue)
                 .Select(result => new
                 {
-                    Id = result.id.Length > 80 ? result.id.Substring(0, 80) : result.id,
+                    Id = result.id.Length > 120 ? result.id.Substring(0, 120) : result.id,
                     DisplayValue = GetRatio(conclusion, result.baseResult, result.diffResult),
                     BaseMedian = result.baseResult.Statistics.Median,
                     DiffMedian = result.diffResult.Statistics.Median,
-                    Modality = GetModalInfo(result.baseResult) ?? GetModalInfo(result.diffResult)
+                    BaseModality = GetModalInfo(result.baseResult),
+                    DiffModality = GetModalInfo(result.diffResult)
                 })
                 .ToArray();
 
@@ -135,7 +136,7 @@ private static void PrintTable((string id, Benchmark baseResult, Benchmark diffR
                 return;
             }
 
-            var table = data.ToMarkdownTable().WithHeaders(conclusion.ToString(), conclusion == EquivalenceTestConclusion.Faster ? "base/diff" : "diff/base", "Base Median (ns)", "Diff Median (ns)", "Modality");
+            var table = data.ToMarkdownTable().WithHeaders(conclusion.ToString(), conclusion == EquivalenceTestConclusion.Faster ? "base/diff" : "diff/base", "Base Median (ns)", "Diff Median (ns)", "Base Modality", "Diff Modality");
 
             foreach (var line in table.ToMarkdown().Split(Environment.NewLine, StringSplitOptions.RemoveEmptyEntries))
                 Console.WriteLine($"| {line.TrimStart()}|"); // the table starts with \t and does not end with '|' and it looks bad so we fix it