Merge pull request dotnet#10468 from basoundr/fixCPC

Fix CPC

src/Test/Perf/infra/automation.csx

@@ -14,8 +14,5 @@ ShellOutVital(Path.Combine(directoryUtil.RoslynDirectory, "Restore.cmd"), "", wo
 // Build Roslyn in Release Mode
 ShellOutVital("msbuild", "./Roslyn.sln /p:Configuration=Release", workingDirectory: directoryUtil.RoslynDirectory);
 
-// Run DownloadTools before using the TraceManager because TraceManager uses the downloaded CPC binaries
-DownloadTools();
-
 // Run run_and_report.csx
 await RunFile(Path.Combine(directoryUtil.MyWorkingDirectory, "run_and_report.csx"));

src/Test/Perf/runner.csx

@@ -9,7 +9,7 @@ using System.IO;
 using System;
 
 var directoryInfo = new RelativeDirectory();
-var testDirectory = Path.Combine(directoryInfo.MyWorkingDirectory, "Tests");
+var testDirectory = Path.Combine(directoryInfo.MyWorkingDirectory, "tests");
 
 // Print message at startup
 Log("Starting Performance Test Run");
@@ -29,45 +29,31 @@ foreach (var script in GetAllCsxRecursive(testDirectory))
 }
 
 var traceManager = TraceManagerFactory.GetTraceManager();
-traceManager.Setup();
-for (int i = 0; i < traceManager.Iterations; i++)
-{
-    traceManager.Start();
-    foreach (dynamic test in testInstances)
-    {
-        test.Setup();
-        traceManager.StartScenario(test.Name, test.MeasuredProc);
-        traceManager.StartEvent();
-        test.Test();
-        traceManager.EndEvent();
-        traceManager.EndScenario();
-    }
-    traceManager.EndScenarios();
-    traceManager.WriteScenariosFileToDisk();
-    traceManager.Stop();
-    traceManager.ResetScenarioGenerator();
-}
-traceManager.Cleanup();
-
-/*
-var traceManager = TraceManagerFactory.GetTraceManager();
-traceManager.Setup();
-// Run each of the tests
-foreach (dynamic test in testInstances) 
+traceManager.Initialize();
+foreach (dynamic test in testInstances)
 {
     test.Setup();
-    traceManager.Start();
-    for (int i = 0; i < test.Iterations; i++) 
+    traceManager.Setup();
+
+    var iterations = traceManager.HasWarmUpIteration ? 
+                     test.Iterations + 1 :
+                     test.Iterations;
+
+    for (int i = 0; i < iterations; i++)
     {
-        traceManager.StartScenario("temp" + i, "csc");
+        traceManager.StartScenarios();
+        traceManager.Start();
+        traceManager.StartScenario(test.Name + i, test.MeasuredProc);
         traceManager.StartEvent();
         test.Test();
         traceManager.EndEvent();
         traceManager.EndScenario();
+
+        traceManager.EndScenarios();
+        traceManager.WriteScenariosFileToDisk();
+        traceManager.Stop();
+        traceManager.ResetScenarioGenerator();
     }
-    traceManager.EndScenarios();
-    traceManager.WriteScenariosFileToDisk();
-    traceManager.Stop();
+
     traceManager.Cleanup();
 }
-*/

src/Test/Perf/tests/csharp/csharp_compiler.csx

@@ -27,9 +27,9 @@ class CSharpCompilerTest: PerfTest
         ShellOutVital(ReleaseCscPath, args, executeInDirectory);
     }
 
-    public override int Iterations => 1;
+    public override int Iterations => 2;
     public override string Name => "csharp " + _rspFile;
-    public override string MeasuredProc => "csc.exe";
+    public override string MeasuredProc => "csc";
 }
 
 TestThisPlease(

src/Test/Perf/tests/helloworld/hello_world.csx

@@ -21,9 +21,9 @@ class HelloWorldTest: PerfTest
         ShellOutVital(ReleaseCscPath, _pathToHelloWorld + " /out:" + _pathToOutput);
     }
 
-    public override int Iterations => 1;
+    public override int Iterations => 2;
     public override string Name => "hello world";
-    public override string MeasuredProc => "csc.exe";
+    public override string MeasuredProc => "csc";
 }
 
 TestThisPlease(new HelloWorldTest());

src/Test/Perf/util/scenario_generator_util.csx

@@ -26,7 +26,6 @@ public class ScenarioGenerator
         }
 
         _buffer = new List<string>();
-        AddScenariosFileStart();
     }
 
     public void AddScenariosFileStart()

src/Test/Perf/util/tools_util.csx

@@ -31,6 +31,7 @@ bool ConvertConsumptionToCsv(string source, string destination, string requiredM
                     if (xmlReader.Name.Equals("ScenarioResult"))
                     {
                         currentScenarioName = xmlReader.GetAttribute("Name");
+                        currentScenarioName = new string(currentScenarioName.TakeWhile(c => !Char.IsDigit(c)).ToArray());
 
                         // These are not test results
                         if (string.Equals(currentScenarioName, "..TestDiagnostics.."))
@@ -132,16 +133,26 @@ void UploadTraces(string sourceFolderPath, string destinationFolderPath)
     Log("Uploading traces");
     if (Directory.Exists(sourceFolderPath))
     {
-        // Get the latest written databackup
-        var directoryToUpload = new DirectoryInfo(sourceFolderPath).GetDirectories("DataBackup*").OrderByDescending(d=>d.LastWriteTimeUtc).FirstOrDefault();
-        if (directoryToUpload == null)
+        var directoriesToUpload = new DirectoryInfo(sourceFolderPath).GetDirectories("DataBackup*");
+        if (directoriesToUpload.Count() == 0)
         {
             Log($"There are no trace directory starting with DataBackup in {sourceFolderPath}");
             return;
         }
 
-        var destination = Path.Combine(destinationFolderPath, directoryToUpload.Name);
-        CopyDirectory(directoryToUpload.FullName, destination);
+        var perfResultDestinationFolderName = string.Format("PerfResults-{0:yyyy-MM-dd_hh-mm-ss-tt}", DateTime.Now);
+
+       var destination = Path.Combine(destinationFolderPath, perfResultDestinationFolderName);
+        foreach (var directoryToUpload in directoriesToUpload)
+        {
+            var destinationDataBackupDirectory = Path.Combine(destination, directoryToUpload.Name);
+            if (Directory.Exists(destinationDataBackupDirectory))
+            {
+                Directory.CreateDirectory(destinationDataBackupDirectory);
+            }
+
+            CopyDirectory(directoryToUpload.FullName, destinationDataBackupDirectory);
+        }
 
         foreach(var file in new DirectoryInfo(sourceFolderPath).GetFiles().Where(f => f.Name.StartsWith("ConsumptionTemp", StringComparison.OrdinalIgnoreCase) || f.Name.StartsWith("Roslyn-", StringComparison.OrdinalIgnoreCase)))
         {

src/Test/Perf/util/trace_manager_util.csx

@@ -7,8 +7,9 @@ using System.IO;
 
 interface ITraceManager
 {
-    int Iterations { get; }
+    bool HasWarmUpIteration { get; }
 
+    void Initialize();
     void Cleanup();
     void EndEvent();
     void EndScenario();
@@ -18,24 +19,25 @@ interface ITraceManager
     void Start();
     void StartEvent();
     void StartScenario(string scenarioName, string processName);
+    void StartScenarios();
     void Stop();
     void WriteScenariosFileToDisk();
 }
 
 class TraceManagerFactory
 {
-    public static ITraceManager GetTraceManager(int iterations = 1)
+    public static ITraceManager GetTraceManager()
     {
         var directoryInfo = new RelativeDirectory();
         var cpcFullPath = Path.Combine(directoryInfo.CPCDirectoryPath, "CPC.exe");
         var scenarioPath = directoryInfo.CPCDirectoryPath;
         if (File.Exists(cpcFullPath))
         {
-            return new TraceManager(iterations, cpcFullPath, scenarioPath);
+            return new TraceManager(cpcFullPath, scenarioPath);
         }
         else
         {
-            return new NoOpTraceManager(iterations);
+            return new NoOpTraceManager();
         }
     }
 }
@@ -44,18 +46,20 @@ class TraceManagerFactory
 /// All operations are NoOp
 class NoOpTraceManager : ITraceManager
 {
-    private readonly int _iterations;
-    public NoOpTraceManager(int iterations)
+    public NoOpTraceManager()
     {
-        _iterations = iterations;
     }
 
-    public int Iterations
+    public bool HasWarmUpIteration
     {
         get
         {
-            return _iterations;
-        }
+            return false;  
+        }    
+    }
+
+    public void Initialize()
+    {
     }
 
     public void Cleanup()
@@ -90,6 +94,10 @@ class NoOpTraceManager : ITraceManager
     {
     }
 
+    public void StartScenarios()
+    {
+    }
+
     public void StartScenario(string scenarioName, string processName)
     {
     }
@@ -106,7 +114,6 @@ class NoOpTraceManager : ITraceManager
 class TraceManager: ITraceManager
 {
     private readonly ScenarioGenerator _scenarioGenerator;
-    private readonly int _iterations;
     private readonly string _cpcPath;
 
     private RelativeDirectory _directoryInfo = new RelativeDirectory();
@@ -115,20 +122,37 @@ class TraceManager: ITraceManager
     private int _stopEventAbsoluteInstance = 1;
 
     public TraceManager(
-        int iterations,
         string cpcPath,
         string scenarioPath): base()
     {
-        _iterations = iterations;
         _cpcPath = cpcPath;
         _scenarioGenerator = new ScenarioGenerator(scenarioPath);
     }
 
-    public int Iterations
+    public bool HasWarmUpIteration
     {
         get
         {
-            return _iterations;
+            return true;
+        }    
+    }
+
+    // Cleanup the results directory and files before every run
+    public void Initialize()
+    {
+        var consumptionTempResultsPath = Path.Combine(_directoryInfo.CPCDirectoryPath, "ConsumptionTempResults.xml");
+        if (File.Exists(consumptionTempResultsPath))
+        {
+            File.Delete(consumptionTempResultsPath);
+        }
+
+        if (Directory.Exists(_directoryInfo.CPCDirectoryPath))
+        {
+            var databackDirectories = Directory.GetDirectories(_directoryInfo.CPCDirectoryPath, "DataBackup*", SearchOption.AllDirectories);
+            foreach (var databackDirectory in databackDirectories)
+            {
+                Directory.Delete(databackDirectory, true);
+            }
         }
     }
 
@@ -145,14 +169,19 @@ class TraceManager: ITraceManager
     public void Stop()
     {
         var scenariosXmlPath = Path.Combine(_directoryInfo.CPCDirectoryPath, "scenarios.xml");
-        var consumptionTempResultsPath = Path.Combine(_directoryInfo.CPCDirectoryPath, "ConsumptionTempResultsPath.xml");
+        var consumptionTempResultsPath = Path.Combine(_directoryInfo.CPCDirectoryPath, "ConsumptionTempResults.xml");
         ShellOutVital(_cpcPath, $"/Stop /DisableArchive /ScenarioPath=\"{scenariosXmlPath}\" /ConsumptionTempResultsPath=\"{consumptionTempResultsPath}\"");
     }
 
     public void Cleanup()
     {
         ShellOutVital(_cpcPath, "/Cleanup /DisableArchive");
     }
+
+    public void StartScenarios()
+    {
+        _scenarioGenerator.AddScenariosFileStart();
+    }
 
     public void StartScenario(string scenarioName, string processName)
     {