Port BenchmarksGame updates to release/2.0.0

.gitattributes

@@ -69,8 +69,10 @@ src/pal/tests/palsuite/paltestlist_to_be_reviewed.txt text eol=lf
 
 tests/src/JIT/Performance/CodeQuality/BenchmarksGame/regexdna/regexdna-input25.txt text eol=lf
 tests/src/JIT/Performance/CodeQuality/BenchmarksGame/regexdna/regexdna-input25000.txt text eol=lf
-tests/src/JIT/Performance/CodeQuality/BenchmarksGame/revcomp/revcomp-input25.txt text eol=lf
-tests/src/JIT/Performance/CodeQuality/BenchmarksGame/revcomp/revcomp-input25000.txt text eol=lf
+tests/src/JIT/Performance/CodeQuality/BenchmarksGame/regex-redux/regexdna-input25.txt text eol=lf
+tests/src/JIT/Performance/CodeQuality/BenchmarksGame/regex-redux/regexdna-input25000.txt text eol=lf
+tests/src/JIT/Performance/CodeQuality/BenchmarksGame/reverse-complement/revcomp-input25.txt text eol=lf
+tests/src/JIT/Performance/CodeQuality/BenchmarksGame/reverse-complement/revcomp-input25000.txt text eol=lf
 tests/src/JIT/Performance/CodeQuality/BenchmarksGame/k-nucleotide/knucleotide-input.txt text eol=lf
 tests/src/JIT/Performance/CodeQuality/BenchmarksGame/k-nucleotide/knucleotide-input-big.txt text eol=lf
 

tests/issues.targets

@@ -393,13 +393,22 @@
         <ExcludeList Include="$(XunitTestBinBase)\JIT\superpmi\superpmicollect\superpmicollect.cmd">
             <Issue>needs triage</Issue>
         </ExcludeList>
-        <ExcludeList Include="$(XunitTestBinBase)\JIT\Performance\CodeQuality\BenchmarksGame\k-nucleotide\k-nucleotide\k-nucleotide.cmd">
+        <ExcludeList Include="$(XunitTestBinBase)\JIT\Performance\CodeQuality\BenchmarksGame\k-nucleotide\k-nucleotide-1\k-nucleotide-1.cmd">
             <Issue>9314</Issue>
         </ExcludeList>
-        <ExcludeList Include="$(XunitTestBinBase)\JIT\Performance\CodeQuality\BenchmarksGame\regexdna\regexdna\regexdna.cmd">
+        <ExcludeList Include="$(XunitTestBinBase)\JIT\Performance\CodeQuality\BenchmarksGame\k-nucleotide\k-nucleotide-9\k-nucleotide-9.cmd">
             <Issue>9314</Issue>
         </ExcludeList>
-        <ExcludeList Include="$(XunitTestBinBase)\JIT\Performance\CodeQuality\BenchmarksGame\revcomp\revcomp\revcomp.cmd">
+        <ExcludeList Include="$(XunitTestBinBase)\JIT\Performance\CodeQuality\BenchmarksGame\regex-redux\regex-redux-1\regex-redux-1.cmd">
+            <Issue>9314</Issue>
+        </ExcludeList>
+        <ExcludeList Include="$(XunitTestBinBase)\JIT\Performance\CodeQuality\BenchmarksGame\regex-redux\regex-redux-5\regex-redux-5.cmd">
+            <Issue>9314</Issue>
+        </ExcludeList>
+        <ExcludeList Include="$(XunitTestBinBase)\JIT\Performance\CodeQuality\BenchmarksGame\reverse-complement\reverse-complement-1\reverse-complement-1.cmd">
+            <Issue>9314</Issue>
+        </ExcludeList>
+        <ExcludeList Include="$(XunitTestBinBase)\JIT\Performance\CodeQuality\BenchmarksGame\reverse-complement\reverse-complement-6\reverse-complement-6.cmd">
             <Issue>9314</Issue>
         </ExcludeList>
     </ItemGroup>

tests/src/JIT/Performance/CodeQuality/BenchmarksGame/README.TXT

@@ -5,9 +5,13 @@ The benchmarks in these sub-directories are based on
 
 See the adjoining LICENSE.TXT file for license terms.
 
-Our intention with these tests is to provide interesting test cases
-for jit developers to use in daily development practice -- not to
-produce variants that give the maximum possible performance.
+Our intention with these tests is twofold:
+ 1 - To track .NET Core's performance on these benchmarks in the
+     same benchmarking system used for other internal .NET Core
+     performance benchmarks.
+ 2 - To make these available for daily JIT (and runtime) development,
+     as a factor in assessing the performance impact of compiler
+     (and runtime) changes.
 
 The benchmarks have been modified to fit into the CoreCLR test and
 performance test framework, as follows:
@@ -19,19 +23,15 @@ performance test framework, as follows:
    xunit-performance iteration is approximately 1 second on modern x64
    hardware
  - reducing verbosity when run as a benchmark
- - reformatting (via the codeformatter tool)
+ - calling different APIs in a few places to allow compiling against
+   netstandard1.4
+ - reformatting
+ - in the case of pidigits, implementing on top of .NET's BigInteger
+   type rather than p/invokes to the native GMP library
 
 These benchmarks are just a subset of the benchmarks available in C# from
-the Benchmarks Game site. We've selected variants that do not rely on
-multiple threads to ensure relative benchmark stability across a
-variety of machines. 
-
-We've excluded two benchmarks that are inherently multitheaded:
-chamenosredux and threadring. We may revisit this as we improve our
-ability to harness threading tests in a stable way.
-
-We've also excluded benchmarks that read in large input files:
-knucleotide, regexdna, revcomp.
-
-
-
+the Benchmarks Game site. The highest-scoring C# .NET Core variant of each
+benchmark is included, and in the (common) case of benchmarks where the
+best-scoring variant uses multiple threads, we've also selected variants
+that do not rely on multiple threads, to ensure relative benchmark stability
+across a variety of machines.

tests/src/JIT/Performance/CodeQuality/BenchmarksGame/binarytrees/binarytrees-2.cs

@@ -0,0 +1,122 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+// Adapted from binary-trees C# .NET Core #2 program
+// http://benchmarksgame.alioth.debian.org/u64q/program.php?test=binarytrees&lang=csharpcore&id=2
+// aka (as of 2017-09-01) rev 1.3 of https://alioth.debian.org/scm/viewvc.php/benchmarksgame/bench/binarytrees/binarytrees.csharp-2.csharp?root=benchmarksgame&view=log
+// Best-scoring single-threaded C# .NET Core version as of 2017-09-01
+
+/* The Computer Language Benchmarks Game
+   http://benchmarksgame.alioth.debian.org/ 
+
+   contributed by Marek Safar 
+   *reset* 
+*/
+
+using System;
+using Microsoft.Xunit.Performance;
+
+[assembly: OptimizeForBenchmarks]
+[assembly: MeasureGCCounts]
+
+namespace BenchmarksGame
+{
+    public class BinaryTrees_2
+    {
+        const int minDepth = 4;
+
+        public static int Main(String[] args)
+        {
+            int n = 0;
+            if (args.Length > 0) n = Int32.Parse(args[0]);
+
+            int check = Bench(n, true);
+            int expected = 4398;
+
+            // Return 100 on success, anything else on failure.
+            return check - expected + 100;
+        }
+
+        [Benchmark(InnerIterationCount = 7)]
+        public static void RunBench()
+        {
+            Benchmark.Iterate(() => Bench(16, false));
+        }
+
+        static int Bench(int n, bool verbose)
+        {
+            int maxDepth = Math.Max(minDepth + 2, n);
+            int stretchDepth = maxDepth + 1;
+
+            int check = (TreeNode.bottomUpTree(stretchDepth)).itemCheck();
+            int checkSum = check;
+            if (verbose) Console.WriteLine("stretch tree of depth {0}\t check: {1}", stretchDepth, check);
+
+            TreeNode longLivedTree = TreeNode.bottomUpTree(maxDepth);
+
+            for (int depth = minDepth; depth <= maxDepth; depth += 2)
+            {
+                int iterations = 1 << (maxDepth - depth + minDepth);
+
+                check = 0;
+                for (int i = 1; i <= iterations; i++)
+                {
+                    check += (TreeNode.bottomUpTree(depth)).itemCheck();
+                }
+                checkSum += check;
+
+                if (verbose)
+                    Console.WriteLine("{0}\t trees of depth {1}\t check: {2}", iterations, depth, check);
+            }
+
+            check = longLivedTree.itemCheck();
+            checkSum += check;
+
+            if (verbose)
+                Console.WriteLine("long lived tree of depth {0}\t check: {1}", maxDepth, check);
+
+            return checkSum;
+        }
+
+
+        struct TreeNode
+        {
+            class Next
+            {
+                public TreeNode left, right;
+            }
+
+            private Next next;
+
+            internal static TreeNode bottomUpTree(int depth)
+            {
+                if (depth > 0)
+                {
+                    return new TreeNode(
+                         bottomUpTree(depth - 1)
+                       , bottomUpTree(depth - 1)
+                       );
+                }
+                else
+                {
+                    return new TreeNode();
+                }
+            }
+
+            TreeNode(TreeNode left, TreeNode right)
+            {
+                this.next = new Next();
+                this.next.left = left;
+                this.next.right = right;
+            }
+
+            internal int itemCheck()
+            {
+                // if necessary deallocate here
+                if (next == null) return 1;
+                else return 1 + next.left.itemCheck() + next.right.itemCheck();
+            }
+        }
+    }
+}

tests/src/JIT/Performance/CodeQuality/BenchmarksGame/binarytrees/binarytrees.csproj → tests/src/JIT/Performance/CodeQuality/BenchmarksGame/binarytrees/binarytrees-2.csproj

@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="utf-8"?>
+<?xml version="1.0" encoding="utf-8"?>
 <Project ToolsVersion="12.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), dir.props))\dir.props" />
   <PropertyGroup>
@@ -31,7 +31,7 @@
     <Service Include="{82A7F48D-3B50-4B1E-B82E-3ADA8210C358}" />
   </ItemGroup>
   <ItemGroup>
-    <Compile Include="binarytrees.csharp.cs" />
+    <Compile Include="$(MSBuildProjectName).cs" />
   </ItemGroup>
   <Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), dir.targets))\dir.targets" />
   <PropertyGroup Condition=" '$(MsBuildProjectDirOverride)' != '' "></PropertyGroup>

tests/src/JIT/Performance/CodeQuality/BenchmarksGame/binarytrees/binarytrees-5.cs

@@ -0,0 +1,147 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+// Adapted from binary-trees C# .NET Core #5 program
+// http://benchmarksgame.alioth.debian.org/u64q/program.php?test=binarytrees&lang=csharpcore&id=5
+// aka (as of 2017-09-01) rev 1.1 of https://alioth.debian.org/scm/viewvc.php/benchmarksgame/bench/binarytrees/binarytrees.csharp-5.csharp?root=benchmarksgame&view=log
+// Best-scoring C# .NET Core version as of 2017-09-01
+
+/* The Computer Language Benchmarks Game
+   http://benchmarksgame.alioth.debian.org/ 
+
+   contributed by Marek Safar  
+   *reset*
+   concurrency added by Peperud
+   minor improvements by Alex Yakunin
+*/
+
+using System;
+using System.Runtime.CompilerServices;
+using System.Threading.Tasks;
+using Microsoft.Xunit.Performance;
+
+[assembly: OptimizeForBenchmarks]
+[assembly: MeasureGCCounts]
+
+namespace BenchmarksGame
+{
+    public sealed class BinaryTrees_5
+    {
+        public const int MinDepth = 4;
+
+        public static int Main(string[] args)
+        {
+            var n = args.Length == 0 ? 0 : int.Parse(args[0]);
+
+            int check = Bench(n, true);
+            int expected = 4398;
+
+            // Return 100 on success, anything else on failure.
+            return check - expected + 100;
+        }
+
+        [Benchmark(InnerIterationCount = 7)]
+        public static void RunBench()
+        {
+            Benchmark.Iterate(() => Bench(16, false));
+        }
+
+        static int Bench(int n, bool verbose)
+        {
+            var maxDepth = n < (MinDepth + 2) ? MinDepth + 2 : n;
+            var stretchDepth = maxDepth + 1;
+
+            var stretchDepthTask = Task.Run(() => TreeNode.CreateTree(stretchDepth).CountNodes());
+            var maxDepthTask = Task.Run(() => TreeNode.CreateTree(maxDepth).CountNodes());
+
+            var tasks = new Task<string>[(maxDepth - MinDepth) / 2 + 1];
+            for (int depth = MinDepth, ti = 0; depth <= maxDepth; depth += 2, ti++)
+            {
+                var iterationCount = 1 << (maxDepth - depth + MinDepth);
+                var depthCopy = depth; // To make sure closure value doesn't change
+                tasks[ti] = Task.Run(() =>
+                {
+                    var count = 0;
+                    if (depthCopy >= 17)
+                    {
+                        // Parallelized computation for relatively large tasks
+                        var miniTasks = new Task<int>[iterationCount];
+                        for (var i = 0; i < iterationCount; i++)
+                            miniTasks[i] = Task.Run(() => TreeNode.CreateTree(depthCopy).CountNodes());
+                        Task.WaitAll(miniTasks);
+                        for (var i = 0; i < iterationCount; i++)
+                            count += miniTasks[i].Result;
+                    }
+                    else
+                    {
+                        // Sequential computation for smaller tasks
+                        for (var i = 0; i < iterationCount; i++)
+                            count += TreeNode.CreateTree(depthCopy).CountNodes();
+                    }
+                    return $"{iterationCount}\t trees of depth {depthCopy}\t check: {count}";
+                });
+            }
+            Task.WaitAll(tasks);
+
+            if (verbose)
+            {
+                int count = 0;
+                Action<string> printAndSum = (string s) =>
+                {
+                    Console.WriteLine(s);
+                    count += int.Parse(s.Substring(s.LastIndexOf(':') + 1).TrimStart());
+                };
+
+                printAndSum(String.Format("stretch tree of depth {0}\t check: {1}",
+                    stretchDepth, stretchDepthTask.Result));
+                foreach (var task in tasks)
+                    printAndSum(task.Result);
+                printAndSum(String.Format("long lived tree of depth {0}\t check: {1}",
+                    maxDepth, maxDepthTask.Result));
+
+                return count;
+            }
+
+            return 0;
+        }
+    }
+
+    public struct TreeNode
+    {
+        public sealed class NodeData
+        {
+            public TreeNode Left, Right;
+
+            public NodeData(TreeNode left, TreeNode right)
+            {
+                Left = left;
+                Right = right;
+            }
+        }
+
+        public NodeData Data;
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public TreeNode(TreeNode left, TreeNode right)
+        {
+            Data = new NodeData(left, right);
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static TreeNode CreateTree(int depth)
+        {
+            return depth <= 0
+                ? default(TreeNode)
+                : new TreeNode(CreateTree(depth - 1), CreateTree(depth - 1));
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public int CountNodes()
+        {
+            if (ReferenceEquals(Data, null))
+                return 1;
+            return 1 + Data.Left.CountNodes() + Data.Right.CountNodes();
+        }
+    }
+}

tests/src/JIT/Performance/CodeQuality/BenchmarksGame/binarytrees/binarytrees3.csproj → tests/src/JIT/Performance/CodeQuality/BenchmarksGame/binarytrees/binarytrees-5.csproj

@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="utf-8"?>
+<?xml version="1.0" encoding="utf-8"?>
 <Project ToolsVersion="12.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), dir.props))\dir.props" />
   <PropertyGroup>
@@ -31,7 +31,7 @@
     <Service Include="{82A7F48D-3B50-4B1E-B82E-3ADA8210C358}" />
   </ItemGroup>
   <ItemGroup>
-    <Compile Include="binarytrees.csharp3.cs" />
+    <Compile Include="$(MSBuildProjectName).cs" />
   </ItemGroup>
   <Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), dir.targets))\dir.targets" />
   <PropertyGroup Condition=" '$(MsBuildProjectDirOverride)' != '' "></PropertyGroup>