From e67077c1635d30f94b927ca42da94b373bceb1dd Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Fri, 14 Jan 2022 10:09:47 -0800 Subject: [PATCH 01/14] Setup test pipeline for closest possible setup to what is currently working for android startup with the new adb method being used. --- eng/pipelines/coreclr/perf.yml | 610 +++++++++--------- .../performance/android_scenarios.proj | 22 +- eng/testing/performance/performance-setup.ps1 | 2 +- 3 files changed, 320 insertions(+), 314 deletions(-) diff --git a/eng/pipelines/coreclr/perf.yml b/eng/pipelines/coreclr/perf.yml index ec233f348b0ecb..b514bcae82f9bd 100644 --- a/eng/pipelines/coreclr/perf.yml +++ b/eng/pipelines/coreclr/perf.yml @@ -114,61 +114,61 @@ jobs: - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'Schedule')) }}: - # build coreclr and libraries - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml - buildConfig: release - platforms: - - Linux_x64 - - windows_x64 - - windows_x86 - - Linux_musl_x64 - jobParameters: - testGroup: perf + ## build coreclr and libraries + #- template: /eng/pipelines/common/platform-matrix.yml + # parameters: + # jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml + # buildConfig: release + # platforms: + # - Linux_x64 + # - windows_x64 + # - windows_x86 + # - Linux_musl_x64 + # jobParameters: + # testGroup: perf - # build mono on wasm - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/global-build-job.yml - buildConfig: release - runtimeFlavor: mono - platforms: - - Browser_wasm - jobParameters: - buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) - nameSuffix: wasm - isOfficialBuild: false - extraStepsTemplate: /eng/pipelines/common/upload-artifact-step.yml - extraStepsParameters: - rootFolder: '$(Build.SourcesDirectory)/artifacts/' - includeRootFolder: true - displayName: Browser Wasm Artifacts - artifactName: BrowserWasm - archiveType: zip - archiveExtension: .zip + ## build mono on wasm + #- template: /eng/pipelines/common/platform-matrix.yml + # parameters: + # jobTemplate: /eng/pipelines/common/global-build-job.yml + # buildConfig: release + # runtimeFlavor: mono + # platforms: + # - Browser_wasm + # jobParameters: + # buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) + # nameSuffix: wasm + # isOfficialBuild: false + # extraStepsTemplate: /eng/pipelines/common/upload-artifact-step.yml + # extraStepsParameters: + # rootFolder: '$(Build.SourcesDirectory)/artifacts/' + # includeRootFolder: true + # displayName: Browser Wasm Artifacts + # artifactName: BrowserWasm + # archiveType: zip + # archiveExtension: .zip - # build mono for AOT - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/global-build-job.yml - buildConfig: release - runtimeFlavor: mono - platforms: - - Linux_x64 - jobParameters: - buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) - nameSuffix: AOT - isOfficialBuild: false - extraStepsTemplate: /eng/pipelines/common/upload-artifact-step.yml - extraStepsParameters: - rootFolder: '$(Build.SourcesDirectory)/artifacts/' - includeRootFolder: true - displayName: AOT Mono Artifacts - artifactName: LinuxMonoAOTx64 - archiveExtension: '.tar.gz' - archiveType: tar - tarCompression: gz + ## build mono for AOT + #- template: /eng/pipelines/common/platform-matrix.yml + # parameters: + # jobTemplate: /eng/pipelines/common/global-build-job.yml + # buildConfig: release + # runtimeFlavor: mono + # platforms: + # - Linux_x64 + # jobParameters: + # buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) + # nameSuffix: AOT + # isOfficialBuild: false + # extraStepsTemplate: /eng/pipelines/common/upload-artifact-step.yml + # extraStepsParameters: + # rootFolder: '$(Build.SourcesDirectory)/artifacts/' + # includeRootFolder: true + # displayName: AOT Mono Artifacts + # artifactName: LinuxMonoAOTx64 + # archiveExtension: '.tar.gz' + # archiveType: tar + # tarCompression: gz # build mono Android scenarios - template: /eng/pipelines/common/platform-matrix.yml @@ -192,27 +192,27 @@ jobs: archiveType: tar tarCompression: gz - # build mono iOS scenarios - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/common/global-build-job.yml - buildConfig: release - runtimeFlavor: mono - platforms: - - iOS_arm64 - jobParameters: - buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) - nameSuffix: iOSMono - isOfficialBuild: false - extraStepsTemplate: /eng/pipelines/coreclr/templates/build-perf-sample-apps.yml - extraStepsParameters: - rootFolder: '$(Build.SourcesDirectory)/artifacts/' - includeRootFolder: true - displayName: iOS Mono Artifacts - artifactName: iOSMonoarm64 - archiveExtension: '.tar.gz' - archiveType: tar - tarCompression: gz + ## build mono iOS scenarios + #- template: /eng/pipelines/common/platform-matrix.yml + # parameters: + # jobTemplate: /eng/pipelines/common/global-build-job.yml + # buildConfig: release + # runtimeFlavor: mono + # platforms: + # - iOS_arm64 + # jobParameters: + # buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) + # nameSuffix: iOSMono + # isOfficialBuild: false + # extraStepsTemplate: /eng/pipelines/coreclr/templates/build-perf-sample-apps.yml + # extraStepsParameters: + # rootFolder: '$(Build.SourcesDirectory)/artifacts/' + # includeRootFolder: true + # displayName: iOS Mono Artifacts + # artifactName: iOSMonoarm64 + # archiveExtension: '.tar.gz' + # archiveType: tar + # tarCompression: gz # build mono - template: /eng/pipelines/common/platform-matrix.yml @@ -239,250 +239,250 @@ jobs: runJobTemplate: /eng/pipelines/coreclr/templates/run-scenarios-job.yml logicalmachine: 'perfpixel4a' - # run mono iOS scenarios and maui iOS scenarios - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml - buildConfig: release - runtimeFlavor: mono - platforms: - - Windows_x64 - jobParameters: - testGroup: perf - runtimeType: iOSMono - projectFile: ios_scenarios.proj - runKind: ios_scenarios - runJobTemplate: /eng/pipelines/coreclr/templates/run-scenarios-job.yml - logicalmachine: 'perfpixel4a' - iosLlvmBuild: False + ## run mono iOS scenarios and maui iOS scenarios + #- template: /eng/pipelines/common/platform-matrix.yml + # parameters: + # jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml + # buildConfig: release + # runtimeFlavor: mono + # platforms: + # - Windows_x64 + # jobParameters: + # testGroup: perf + # runtimeType: iOSMono + # projectFile: ios_scenarios.proj + # runKind: ios_scenarios + # runJobTemplate: /eng/pipelines/coreclr/templates/run-scenarios-job.yml + # logicalmachine: 'perfpixel4a' + # iosLlvmBuild: False - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml - buildConfig: release - runtimeFlavor: mono - platforms: - - Windows_x64 - jobParameters: - testGroup: perf - runtimeType: iOSMono - projectFile: ios_scenarios.proj - runKind: ios_scenarios - runJobTemplate: /eng/pipelines/coreclr/templates/run-scenarios-job.yml - logicalmachine: 'perfpixel4a' - iosLlvmBuild: True + #- template: /eng/pipelines/common/platform-matrix.yml + # parameters: + # jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml + # buildConfig: release + # runtimeFlavor: mono + # platforms: + # - Windows_x64 + # jobParameters: + # testGroup: perf + # runtimeType: iOSMono + # projectFile: ios_scenarios.proj + # runKind: ios_scenarios + # runJobTemplate: /eng/pipelines/coreclr/templates/run-scenarios-job.yml + # logicalmachine: 'perfpixel4a' + # iosLlvmBuild: True - # run mono microbenchmarks perf job - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml - buildConfig: release - runtimeFlavor: mono - platforms: - - Linux_x64 - jobParameters: - testGroup: perf - liveLibrariesBuildConfig: Release - runtimeType: mono - projectFile: microbenchmarks.proj - runKind: micro_mono - runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml - logicalmachine: 'perftiger' + ## run mono microbenchmarks perf job + #- template: /eng/pipelines/common/platform-matrix.yml + # parameters: + # jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml + # buildConfig: release + # runtimeFlavor: mono + # platforms: + # - Linux_x64 + # jobParameters: + # testGroup: perf + # liveLibrariesBuildConfig: Release + # runtimeType: mono + # projectFile: microbenchmarks.proj + # runKind: micro_mono + # runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml + # logicalmachine: 'perftiger' - # run mono interpreter perf job - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml - buildConfig: release - runtimeFlavor: mono - platforms: - - Linux_x64 - jobParameters: - testGroup: perf - liveLibrariesBuildConfig: Release - runtimeType: mono - codeGenType: 'Interpreter' - projectFile: microbenchmarks.proj - runKind: micro_mono - runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml - logicalmachine: 'perftiger' + ## run mono interpreter perf job + #- template: /eng/pipelines/common/platform-matrix.yml + # parameters: + # jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml + # buildConfig: release + # runtimeFlavor: mono + # platforms: + # - Linux_x64 + # jobParameters: + # testGroup: perf + # liveLibrariesBuildConfig: Release + # runtimeType: mono + # codeGenType: 'Interpreter' + # projectFile: microbenchmarks.proj + # runKind: micro_mono + # runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml + # logicalmachine: 'perftiger' - # run mono wasm interpreter (default) microbenchmarks perf job - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml # NOTE: should we move this file out of coreclr tempelates because it contains mono jobs? - buildConfig: release - runtimeFlavor: wasm - platforms: - - Linux_x64 - jobParameters: - testGroup: perf - liveLibrariesBuildConfig: Release - runtimeType: wasm - codeGenType: 'wasm' - projectFile: microbenchmarks.proj - runKind: micro - runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml - logicalmachine: 'perftiger' - javascriptEngine: 'v8' + ## run mono wasm interpreter (default) microbenchmarks perf job + #- template: /eng/pipelines/common/platform-matrix.yml + # parameters: + # jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml # NOTE: should we move this file out of coreclr tempelates because it contains mono jobs? + # buildConfig: release + # runtimeFlavor: wasm + # platforms: + # - Linux_x64 + # jobParameters: + # testGroup: perf + # liveLibrariesBuildConfig: Release + # runtimeType: wasm + # codeGenType: 'wasm' + # projectFile: microbenchmarks.proj + # runKind: micro + # runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml + # logicalmachine: 'perftiger' + # javascriptEngine: 'v8' - #run mono wasm aot microbenchmarks perf job - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobtemplate: /eng/pipelines/coreclr/templates/perf-job.yml # note: should we move this file out of coreclr tempelates because it contains mono jobs? - buildconfig: release - runtimeflavor: wasm - platforms: - - linux_x64 - jobparameters: - testgroup: perf - livelibrariesbuildconfig: Release - runtimetype: wasm - codegentype: 'aot' - projectfile: microbenchmarks.proj - runkind: micro - runjobtemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml - logicalmachine: 'perftiger' - javascriptEngine: 'v8' + ##run mono wasm aot microbenchmarks perf job + #- template: /eng/pipelines/common/platform-matrix.yml + # parameters: + # jobtemplate: /eng/pipelines/coreclr/templates/perf-job.yml # note: should we move this file out of coreclr tempelates because it contains mono jobs? + # buildconfig: release + # runtimeflavor: wasm + # platforms: + # - linux_x64 + # jobparameters: + # testgroup: perf + # livelibrariesbuildconfig: Release + # runtimetype: wasm + # codegentype: 'aot' + # projectfile: microbenchmarks.proj + # runkind: micro + # runjobtemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml + # logicalmachine: 'perftiger' + # javascriptEngine: 'v8' - # run mono aot microbenchmarks perf job - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml # NOTE: should we move this file out of coreclr tempelates because it contains mono jobs? - buildConfig: release - runtimeFlavor: aot - platforms: - - Linux_x64 - jobParameters: - testGroup: perf - liveLibrariesBuildConfig: Release - runtimeType: mono - codeGenType: 'AOT' - projectFile: microbenchmarks.proj - runKind: micro_mono - runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml - logicalmachine: 'perftiger' + ## run mono aot microbenchmarks perf job + #- template: /eng/pipelines/common/platform-matrix.yml + # parameters: + # jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml # NOTE: should we move this file out of coreclr tempelates because it contains mono jobs? + # buildConfig: release + # runtimeFlavor: aot + # platforms: + # - Linux_x64 + # jobParameters: + # testGroup: perf + # liveLibrariesBuildConfig: Release + # runtimeType: mono + # codeGenType: 'AOT' + # projectFile: microbenchmarks.proj + # runKind: micro_mono + # runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml + # logicalmachine: 'perftiger' - # run coreclr perftiger microbenchmarks perf job - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml - buildConfig: release - runtimeFlavor: coreclr - platforms: - - Linux_x64 - - windows_x64 - - windows_x86 - - Linux_musl_x64 - jobParameters: - testGroup: perf - liveLibrariesBuildConfig: Release - projectFile: microbenchmarks.proj - runKind: micro - runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml - logicalmachine: 'perftiger' + ## run coreclr perftiger microbenchmarks perf job + #- template: /eng/pipelines/common/platform-matrix.yml + # parameters: + # jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml + # buildConfig: release + # runtimeFlavor: coreclr + # platforms: + # - Linux_x64 + # - windows_x64 + # - windows_x86 + # - Linux_musl_x64 + # jobParameters: + # testGroup: perf + # liveLibrariesBuildConfig: Release + # projectFile: microbenchmarks.proj + # runKind: micro + # runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml + # logicalmachine: 'perftiger' - # run coreclr perftiger microbenchmarks pgo perf jobs - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml - buildConfig: release - runtimeFlavor: coreclr - platforms: - - windows_x64 - jobParameters: - testGroup: perf - liveLibrariesBuildConfig: Release - projectFile: microbenchmarks.proj - runKind: micro - runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml - logicalmachine: 'perftiger' - pgoRunType: -NoPgo + ## run coreclr perftiger microbenchmarks pgo perf jobs + #- template: /eng/pipelines/common/platform-matrix.yml + # parameters: + # jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml + # buildConfig: release + # runtimeFlavor: coreclr + # platforms: + # - windows_x64 + # jobParameters: + # testGroup: perf + # liveLibrariesBuildConfig: Release + # projectFile: microbenchmarks.proj + # runKind: micro + # runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml + # logicalmachine: 'perftiger' + # pgoRunType: -NoPgo - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml - buildConfig: release - runtimeFlavor: coreclr - platforms: - - windows_x64 - jobParameters: - testGroup: perf - liveLibrariesBuildConfig: Release - projectFile: microbenchmarks.proj - runKind: micro - runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml - logicalmachine: 'perftiger' - pgoRunType: -DynamicPgo + #- template: /eng/pipelines/common/platform-matrix.yml + # parameters: + # jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml + # buildConfig: release + # runtimeFlavor: coreclr + # platforms: + # - windows_x64 + # jobParameters: + # testGroup: perf + # liveLibrariesBuildConfig: Release + # projectFile: microbenchmarks.proj + # runKind: micro + # runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml + # logicalmachine: 'perftiger' + # pgoRunType: -DynamicPgo - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml - buildConfig: release - runtimeFlavor: coreclr - platforms: - - windows_x64 - jobParameters: - testGroup: perf - liveLibrariesBuildConfig: Release - projectFile: microbenchmarks.proj - runKind: micro - runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml - logicalmachine: 'perftiger' - pgoRunType: -FullPgo + #- template: /eng/pipelines/common/platform-matrix.yml + # parameters: + # jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml + # buildConfig: release + # runtimeFlavor: coreclr + # platforms: + # - windows_x64 + # jobParameters: + # testGroup: perf + # liveLibrariesBuildConfig: Release + # projectFile: microbenchmarks.proj + # runKind: micro + # runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml + # logicalmachine: 'perftiger' + # pgoRunType: -FullPgo - # run coreclr perfowl microbenchmarks perf job - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml - buildConfig: release - runtimeFlavor: coreclr - platforms: - - Linux_x64 - - windows_x64 - jobParameters: - testGroup: perf - liveLibrariesBuildConfig: Release - projectFile: microbenchmarks.proj - runKind: micro - runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml - logicalmachine: 'perfowl' + ## run coreclr perfowl microbenchmarks perf job + #- template: /eng/pipelines/common/platform-matrix.yml + # parameters: + # jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml + # buildConfig: release + # runtimeFlavor: coreclr + # platforms: + # - Linux_x64 + # - windows_x64 + # jobParameters: + # testGroup: perf + # liveLibrariesBuildConfig: Release + # projectFile: microbenchmarks.proj + # runKind: micro + # runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml + # logicalmachine: 'perfowl' - # run coreclr crossgen perf job - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml - buildConfig: release - runtimeFlavor: coreclr - platforms: - - Linux_x64 - - windows_x64 - - windows_x86 - jobParameters: - testGroup: perf - liveLibrariesBuildConfig: Release - projectFile: crossgen_perf.proj - runKind: crossgen_scenarios - runJobTemplate: /eng/pipelines/coreclr/templates/run-scenarios-job.yml - logicalmachine: 'perftiger' + ## run coreclr crossgen perf job + #- template: /eng/pipelines/common/platform-matrix.yml + # parameters: + # jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml + # buildConfig: release + # runtimeFlavor: coreclr + # platforms: + # - Linux_x64 + # - windows_x64 + # - windows_x86 + # jobParameters: + # testGroup: perf + # liveLibrariesBuildConfig: Release + # projectFile: crossgen_perf.proj + # runKind: crossgen_scenarios + # runJobTemplate: /eng/pipelines/coreclr/templates/run-scenarios-job.yml + # logicalmachine: 'perftiger' - # run mono wasm blazor perf job - - template: /eng/pipelines/common/platform-matrix.yml - parameters: - jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml - buildConfig: release - runtimeFlavor: wasm - platforms: - - Linux_x64 - jobParameters: - testGroup: perf - liveLibrariesBuildConfig: Release - runtimeType: wasm - projectFile: blazor_perf.proj - runKind: blazor_scenarios - runJobTemplate: /eng/pipelines/coreclr/templates/run-scenarios-job.yml - additionalSetupParameters: '--latestdotnet' - logicalmachine: 'perftiger' + ## run mono wasm blazor perf job + #- template: /eng/pipelines/common/platform-matrix.yml + # parameters: + # jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml + # buildConfig: release + # runtimeFlavor: wasm + # platforms: + # - Linux_x64 + # jobParameters: + # testGroup: perf + # liveLibrariesBuildConfig: Release + # runtimeType: wasm + # projectFile: blazor_perf.proj + # runKind: blazor_scenarios + # runJobTemplate: /eng/pipelines/coreclr/templates/run-scenarios-job.yml + # additionalSetupParameters: '--latestdotnet' + # logicalmachine: 'perftiger' # build maui workload packs - template: /eng/pipelines/common/platform-matrix.yml diff --git a/eng/testing/performance/android_scenarios.proj b/eng/testing/performance/android_scenarios.proj index 52af8fa1678dc8..3351d136be5c7b 100644 --- a/eng/testing/performance/android_scenarios.proj +++ b/eng/testing/performance/android_scenarios.proj @@ -23,35 +23,41 @@ - + $(WorkItemDirectory) echo on;set XHARNESSPATH=$(XharnessPath);cd $(ScenarioDirectory)helloandroid;copy %HELIX_CORRELATION_PAYLOAD%\HelloAndroidWithDiag.apk .;$(Python) pre.py $(Python) test.py devicestartup --device-type android --package-path pub\HelloAndroidWithDiag.apk --package-name net.dot.HelloAndroid --exit-code 42 --scenario-name "%(Identity)" $(Python) post.py + + $(WorkItemDirectory) + echo on;set XHARNESSPATH=$(XharnessPath);cd $(ScenarioDirectory)hellomaui;copy %HELIX_CORRELATION_PAYLOAD%\MauiAndroidDefault.apk .;$(Python) pre.py + $(Python) test.py devicestartup --device-type android --package-path pub\MauiAndroidDefault.apk --package-name com.companyname.MauiTesting --exit-code 0 --scenario-name "%(Identity)" + $(Python) post.py + diff --git a/eng/testing/performance/performance-setup.ps1 b/eng/testing/performance/performance-setup.ps1 index d85325ee1349d5..72840befe2a03e 100644 --- a/eng/testing/performance/performance-setup.ps1 +++ b/eng/testing/performance/performance-setup.ps1 @@ -124,7 +124,7 @@ if ($RunFromPerformanceRepo) { robocopy $SourceDirectory $PerformanceDirectory /E /XD $PayloadDirectory $SourceDirectory\artifacts $SourceDirectory\.git } else { - git clone --branch main --depth 1 --quiet https://github.com/dotnet/performance $PerformanceDirectory + git clone --branch MinimalChangeStartupADB --depth 1 --quiet https://github.com/LoopedBard3/performance $PerformanceDirectory } if($MonoDotnet -ne "") From e90ab882b6474d9d23e3f3fe92c36f8a266d65c6 Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Fri, 14 Jan 2022 11:44:22 -0800 Subject: [PATCH 02/14] Fix android scenario problems. --- eng/testing/performance/android_scenarios.proj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/testing/performance/android_scenarios.proj b/eng/testing/performance/android_scenarios.proj index 3351d136be5c7b..4b8e9d40c2eee3 100644 --- a/eng/testing/performance/android_scenarios.proj +++ b/eng/testing/performance/android_scenarios.proj @@ -49,13 +49,13 @@ --> $(WorkItemDirectory) - echo on;set XHARNESSPATH=$(XharnessPath);cd $(ScenarioDirectory)helloandroid;copy %HELIX_CORRELATION_PAYLOAD%\HelloAndroidWithDiag.apk .;$(Python) pre.py + echo on;set XHARNESSPATH=$(XharnessPath);cd $(ScenarioDirectory)helloandroid;copy %HELIX_CORRELATION_PAYLOAD%\HelloAndroid.apk .;copy %HELIX_CORRELATION_PAYLOAD%\HelloAndroidWithDiag.apk .;$(Python) pre.py $(Python) test.py devicestartup --device-type android --package-path pub\HelloAndroidWithDiag.apk --package-name net.dot.HelloAndroid --exit-code 42 --scenario-name "%(Identity)" $(Python) post.py $(WorkItemDirectory) - echo on;set XHARNESSPATH=$(XharnessPath);cd $(ScenarioDirectory)hellomaui;copy %HELIX_CORRELATION_PAYLOAD%\MauiAndroidDefault.apk .;$(Python) pre.py + echo on;set XHARNESSPATH=$(XharnessPath);cd $(ScenarioDirectory)mauiandroid;copy %HELIX_CORRELATION_PAYLOAD%\MauiAndroidDefault.apk .;$(Python) pre.py $(Python) test.py devicestartup --device-type android --package-path pub\MauiAndroidDefault.apk --package-name com.companyname.MauiTesting --exit-code 0 --scenario-name "%(Identity)" $(Python) post.py From 11138f8b6dd5a850479a3668c89b5e5f7d621cc7 Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Wed, 19 Jan 2022 13:14:18 -0800 Subject: [PATCH 03/14] Use latest xharness version for testing popen fix. --- eng/testing/performance/android_scenarios.proj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/testing/performance/android_scenarios.proj b/eng/testing/performance/android_scenarios.proj index 4b8e9d40c2eee3..fc1c2b7bffc4ff 100644 --- a/eng/testing/performance/android_scenarios.proj +++ b/eng/testing/performance/android_scenarios.proj @@ -1,8 +1,8 @@ true - 1.0.0-prerelease.21566.2 - %HELIX_CORRELATION_PAYLOAD%\microsoft.dotnet.xharness.cli\$(MicrosoftDotNetXHarnessCLIVersion)\tools\net6.0\any\Microsoft.DotNet.XHarness.CLI.dll + python3 From 1fd795790a2b72f5ab6a9639d21f0b5d5ed186d8 Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Wed, 19 Jan 2022 13:15:41 -0800 Subject: [PATCH 04/14] Wrong branch changed. --- eng/testing/performance/android_scenarios.proj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/testing/performance/android_scenarios.proj b/eng/testing/performance/android_scenarios.proj index fc1c2b7bffc4ff..4b8e9d40c2eee3 100644 --- a/eng/testing/performance/android_scenarios.proj +++ b/eng/testing/performance/android_scenarios.proj @@ -1,8 +1,8 @@ true - + 1.0.0-prerelease.21566.2 + %HELIX_CORRELATION_PAYLOAD%\microsoft.dotnet.xharness.cli\$(MicrosoftDotNetXHarnessCLIVersion)\tools\net6.0\any\Microsoft.DotNet.XHarness.CLI.dll python3 From 95b0476b9db3a1d8df15e24099584ba3aa53e76f Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Thu, 20 Jan 2022 15:18:23 -0800 Subject: [PATCH 05/14] Add -u for unbuffered python IO to see if there is more information missing during the lags. --- eng/testing/performance/android_scenarios.proj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/testing/performance/android_scenarios.proj b/eng/testing/performance/android_scenarios.proj index 4b8e9d40c2eee3..841dc23687cd89 100644 --- a/eng/testing/performance/android_scenarios.proj +++ b/eng/testing/performance/android_scenarios.proj @@ -50,13 +50,13 @@ $(WorkItemDirectory) echo on;set XHARNESSPATH=$(XharnessPath);cd $(ScenarioDirectory)helloandroid;copy %HELIX_CORRELATION_PAYLOAD%\HelloAndroid.apk .;copy %HELIX_CORRELATION_PAYLOAD%\HelloAndroidWithDiag.apk .;$(Python) pre.py - $(Python) test.py devicestartup --device-type android --package-path pub\HelloAndroidWithDiag.apk --package-name net.dot.HelloAndroid --exit-code 42 --scenario-name "%(Identity)" + $(Python) -u test.py devicestartup --device-type android --package-path pub\HelloAndroidWithDiag.apk --package-name net.dot.HelloAndroid --exit-code 42 --scenario-name "%(Identity)" $(Python) post.py $(WorkItemDirectory) echo on;set XHARNESSPATH=$(XharnessPath);cd $(ScenarioDirectory)mauiandroid;copy %HELIX_CORRELATION_PAYLOAD%\MauiAndroidDefault.apk .;$(Python) pre.py - $(Python) test.py devicestartup --device-type android --package-path pub\MauiAndroidDefault.apk --package-name com.companyname.MauiTesting --exit-code 0 --scenario-name "%(Identity)" + $(Python) -u test.py devicestartup --device-type android --package-path pub\MauiAndroidDefault.apk --package-name com.companyname.MauiTesting --exit-code 0 --scenario-name "%(Identity)" $(Python) post.py From 0bbaf66df25e43a21f490aea4c8f211bbb6dc67b Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Fri, 21 Jan 2022 13:41:14 -0800 Subject: [PATCH 06/14] Final run setup for runtime part of the repo. --- eng/pipelines/coreclr/perf.yml | 610 +++++++++--------- .../templates/build-perf-sample-apps.yml | 13 - .../performance/android_scenarios.proj | 22 +- eng/testing/performance/performance-setup.ps1 | 11 +- 4 files changed, 319 insertions(+), 337 deletions(-) diff --git a/eng/pipelines/coreclr/perf.yml b/eng/pipelines/coreclr/perf.yml index b514bcae82f9bd..ec233f348b0ecb 100644 --- a/eng/pipelines/coreclr/perf.yml +++ b/eng/pipelines/coreclr/perf.yml @@ -114,61 +114,61 @@ jobs: - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'Schedule')) }}: - ## build coreclr and libraries - #- template: /eng/pipelines/common/platform-matrix.yml - # parameters: - # jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml - # buildConfig: release - # platforms: - # - Linux_x64 - # - windows_x64 - # - windows_x86 - # - Linux_musl_x64 - # jobParameters: - # testGroup: perf + # build coreclr and libraries + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml + buildConfig: release + platforms: + - Linux_x64 + - windows_x64 + - windows_x86 + - Linux_musl_x64 + jobParameters: + testGroup: perf - ## build mono on wasm - #- template: /eng/pipelines/common/platform-matrix.yml - # parameters: - # jobTemplate: /eng/pipelines/common/global-build-job.yml - # buildConfig: release - # runtimeFlavor: mono - # platforms: - # - Browser_wasm - # jobParameters: - # buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) - # nameSuffix: wasm - # isOfficialBuild: false - # extraStepsTemplate: /eng/pipelines/common/upload-artifact-step.yml - # extraStepsParameters: - # rootFolder: '$(Build.SourcesDirectory)/artifacts/' - # includeRootFolder: true - # displayName: Browser Wasm Artifacts - # artifactName: BrowserWasm - # archiveType: zip - # archiveExtension: .zip + # build mono on wasm + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + buildConfig: release + runtimeFlavor: mono + platforms: + - Browser_wasm + jobParameters: + buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) + nameSuffix: wasm + isOfficialBuild: false + extraStepsTemplate: /eng/pipelines/common/upload-artifact-step.yml + extraStepsParameters: + rootFolder: '$(Build.SourcesDirectory)/artifacts/' + includeRootFolder: true + displayName: Browser Wasm Artifacts + artifactName: BrowserWasm + archiveType: zip + archiveExtension: .zip - ## build mono for AOT - #- template: /eng/pipelines/common/platform-matrix.yml - # parameters: - # jobTemplate: /eng/pipelines/common/global-build-job.yml - # buildConfig: release - # runtimeFlavor: mono - # platforms: - # - Linux_x64 - # jobParameters: - # buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) - # nameSuffix: AOT - # isOfficialBuild: false - # extraStepsTemplate: /eng/pipelines/common/upload-artifact-step.yml - # extraStepsParameters: - # rootFolder: '$(Build.SourcesDirectory)/artifacts/' - # includeRootFolder: true - # displayName: AOT Mono Artifacts - # artifactName: LinuxMonoAOTx64 - # archiveExtension: '.tar.gz' - # archiveType: tar - # tarCompression: gz + # build mono for AOT + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + buildConfig: release + runtimeFlavor: mono + platforms: + - Linux_x64 + jobParameters: + buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) + nameSuffix: AOT + isOfficialBuild: false + extraStepsTemplate: /eng/pipelines/common/upload-artifact-step.yml + extraStepsParameters: + rootFolder: '$(Build.SourcesDirectory)/artifacts/' + includeRootFolder: true + displayName: AOT Mono Artifacts + artifactName: LinuxMonoAOTx64 + archiveExtension: '.tar.gz' + archiveType: tar + tarCompression: gz # build mono Android scenarios - template: /eng/pipelines/common/platform-matrix.yml @@ -192,27 +192,27 @@ jobs: archiveType: tar tarCompression: gz - ## build mono iOS scenarios - #- template: /eng/pipelines/common/platform-matrix.yml - # parameters: - # jobTemplate: /eng/pipelines/common/global-build-job.yml - # buildConfig: release - # runtimeFlavor: mono - # platforms: - # - iOS_arm64 - # jobParameters: - # buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) - # nameSuffix: iOSMono - # isOfficialBuild: false - # extraStepsTemplate: /eng/pipelines/coreclr/templates/build-perf-sample-apps.yml - # extraStepsParameters: - # rootFolder: '$(Build.SourcesDirectory)/artifacts/' - # includeRootFolder: true - # displayName: iOS Mono Artifacts - # artifactName: iOSMonoarm64 - # archiveExtension: '.tar.gz' - # archiveType: tar - # tarCompression: gz + # build mono iOS scenarios + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/common/global-build-job.yml + buildConfig: release + runtimeFlavor: mono + platforms: + - iOS_arm64 + jobParameters: + buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) + nameSuffix: iOSMono + isOfficialBuild: false + extraStepsTemplate: /eng/pipelines/coreclr/templates/build-perf-sample-apps.yml + extraStepsParameters: + rootFolder: '$(Build.SourcesDirectory)/artifacts/' + includeRootFolder: true + displayName: iOS Mono Artifacts + artifactName: iOSMonoarm64 + archiveExtension: '.tar.gz' + archiveType: tar + tarCompression: gz # build mono - template: /eng/pipelines/common/platform-matrix.yml @@ -239,250 +239,250 @@ jobs: runJobTemplate: /eng/pipelines/coreclr/templates/run-scenarios-job.yml logicalmachine: 'perfpixel4a' - ## run mono iOS scenarios and maui iOS scenarios - #- template: /eng/pipelines/common/platform-matrix.yml - # parameters: - # jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml - # buildConfig: release - # runtimeFlavor: mono - # platforms: - # - Windows_x64 - # jobParameters: - # testGroup: perf - # runtimeType: iOSMono - # projectFile: ios_scenarios.proj - # runKind: ios_scenarios - # runJobTemplate: /eng/pipelines/coreclr/templates/run-scenarios-job.yml - # logicalmachine: 'perfpixel4a' - # iosLlvmBuild: False + # run mono iOS scenarios and maui iOS scenarios + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml + buildConfig: release + runtimeFlavor: mono + platforms: + - Windows_x64 + jobParameters: + testGroup: perf + runtimeType: iOSMono + projectFile: ios_scenarios.proj + runKind: ios_scenarios + runJobTemplate: /eng/pipelines/coreclr/templates/run-scenarios-job.yml + logicalmachine: 'perfpixel4a' + iosLlvmBuild: False - #- template: /eng/pipelines/common/platform-matrix.yml - # parameters: - # jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml - # buildConfig: release - # runtimeFlavor: mono - # platforms: - # - Windows_x64 - # jobParameters: - # testGroup: perf - # runtimeType: iOSMono - # projectFile: ios_scenarios.proj - # runKind: ios_scenarios - # runJobTemplate: /eng/pipelines/coreclr/templates/run-scenarios-job.yml - # logicalmachine: 'perfpixel4a' - # iosLlvmBuild: True + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml + buildConfig: release + runtimeFlavor: mono + platforms: + - Windows_x64 + jobParameters: + testGroup: perf + runtimeType: iOSMono + projectFile: ios_scenarios.proj + runKind: ios_scenarios + runJobTemplate: /eng/pipelines/coreclr/templates/run-scenarios-job.yml + logicalmachine: 'perfpixel4a' + iosLlvmBuild: True - ## run mono microbenchmarks perf job - #- template: /eng/pipelines/common/platform-matrix.yml - # parameters: - # jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml - # buildConfig: release - # runtimeFlavor: mono - # platforms: - # - Linux_x64 - # jobParameters: - # testGroup: perf - # liveLibrariesBuildConfig: Release - # runtimeType: mono - # projectFile: microbenchmarks.proj - # runKind: micro_mono - # runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml - # logicalmachine: 'perftiger' + # run mono microbenchmarks perf job + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml + buildConfig: release + runtimeFlavor: mono + platforms: + - Linux_x64 + jobParameters: + testGroup: perf + liveLibrariesBuildConfig: Release + runtimeType: mono + projectFile: microbenchmarks.proj + runKind: micro_mono + runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml + logicalmachine: 'perftiger' - ## run mono interpreter perf job - #- template: /eng/pipelines/common/platform-matrix.yml - # parameters: - # jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml - # buildConfig: release - # runtimeFlavor: mono - # platforms: - # - Linux_x64 - # jobParameters: - # testGroup: perf - # liveLibrariesBuildConfig: Release - # runtimeType: mono - # codeGenType: 'Interpreter' - # projectFile: microbenchmarks.proj - # runKind: micro_mono - # runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml - # logicalmachine: 'perftiger' + # run mono interpreter perf job + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml + buildConfig: release + runtimeFlavor: mono + platforms: + - Linux_x64 + jobParameters: + testGroup: perf + liveLibrariesBuildConfig: Release + runtimeType: mono + codeGenType: 'Interpreter' + projectFile: microbenchmarks.proj + runKind: micro_mono + runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml + logicalmachine: 'perftiger' - ## run mono wasm interpreter (default) microbenchmarks perf job - #- template: /eng/pipelines/common/platform-matrix.yml - # parameters: - # jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml # NOTE: should we move this file out of coreclr tempelates because it contains mono jobs? - # buildConfig: release - # runtimeFlavor: wasm - # platforms: - # - Linux_x64 - # jobParameters: - # testGroup: perf - # liveLibrariesBuildConfig: Release - # runtimeType: wasm - # codeGenType: 'wasm' - # projectFile: microbenchmarks.proj - # runKind: micro - # runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml - # logicalmachine: 'perftiger' - # javascriptEngine: 'v8' + # run mono wasm interpreter (default) microbenchmarks perf job + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml # NOTE: should we move this file out of coreclr tempelates because it contains mono jobs? + buildConfig: release + runtimeFlavor: wasm + platforms: + - Linux_x64 + jobParameters: + testGroup: perf + liveLibrariesBuildConfig: Release + runtimeType: wasm + codeGenType: 'wasm' + projectFile: microbenchmarks.proj + runKind: micro + runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml + logicalmachine: 'perftiger' + javascriptEngine: 'v8' - ##run mono wasm aot microbenchmarks perf job - #- template: /eng/pipelines/common/platform-matrix.yml - # parameters: - # jobtemplate: /eng/pipelines/coreclr/templates/perf-job.yml # note: should we move this file out of coreclr tempelates because it contains mono jobs? - # buildconfig: release - # runtimeflavor: wasm - # platforms: - # - linux_x64 - # jobparameters: - # testgroup: perf - # livelibrariesbuildconfig: Release - # runtimetype: wasm - # codegentype: 'aot' - # projectfile: microbenchmarks.proj - # runkind: micro - # runjobtemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml - # logicalmachine: 'perftiger' - # javascriptEngine: 'v8' + #run mono wasm aot microbenchmarks perf job + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobtemplate: /eng/pipelines/coreclr/templates/perf-job.yml # note: should we move this file out of coreclr tempelates because it contains mono jobs? + buildconfig: release + runtimeflavor: wasm + platforms: + - linux_x64 + jobparameters: + testgroup: perf + livelibrariesbuildconfig: Release + runtimetype: wasm + codegentype: 'aot' + projectfile: microbenchmarks.proj + runkind: micro + runjobtemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml + logicalmachine: 'perftiger' + javascriptEngine: 'v8' - ## run mono aot microbenchmarks perf job - #- template: /eng/pipelines/common/platform-matrix.yml - # parameters: - # jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml # NOTE: should we move this file out of coreclr tempelates because it contains mono jobs? - # buildConfig: release - # runtimeFlavor: aot - # platforms: - # - Linux_x64 - # jobParameters: - # testGroup: perf - # liveLibrariesBuildConfig: Release - # runtimeType: mono - # codeGenType: 'AOT' - # projectFile: microbenchmarks.proj - # runKind: micro_mono - # runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml - # logicalmachine: 'perftiger' + # run mono aot microbenchmarks perf job + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml # NOTE: should we move this file out of coreclr tempelates because it contains mono jobs? + buildConfig: release + runtimeFlavor: aot + platforms: + - Linux_x64 + jobParameters: + testGroup: perf + liveLibrariesBuildConfig: Release + runtimeType: mono + codeGenType: 'AOT' + projectFile: microbenchmarks.proj + runKind: micro_mono + runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml + logicalmachine: 'perftiger' - ## run coreclr perftiger microbenchmarks perf job - #- template: /eng/pipelines/common/platform-matrix.yml - # parameters: - # jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml - # buildConfig: release - # runtimeFlavor: coreclr - # platforms: - # - Linux_x64 - # - windows_x64 - # - windows_x86 - # - Linux_musl_x64 - # jobParameters: - # testGroup: perf - # liveLibrariesBuildConfig: Release - # projectFile: microbenchmarks.proj - # runKind: micro - # runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml - # logicalmachine: 'perftiger' + # run coreclr perftiger microbenchmarks perf job + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml + buildConfig: release + runtimeFlavor: coreclr + platforms: + - Linux_x64 + - windows_x64 + - windows_x86 + - Linux_musl_x64 + jobParameters: + testGroup: perf + liveLibrariesBuildConfig: Release + projectFile: microbenchmarks.proj + runKind: micro + runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml + logicalmachine: 'perftiger' - ## run coreclr perftiger microbenchmarks pgo perf jobs - #- template: /eng/pipelines/common/platform-matrix.yml - # parameters: - # jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml - # buildConfig: release - # runtimeFlavor: coreclr - # platforms: - # - windows_x64 - # jobParameters: - # testGroup: perf - # liveLibrariesBuildConfig: Release - # projectFile: microbenchmarks.proj - # runKind: micro - # runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml - # logicalmachine: 'perftiger' - # pgoRunType: -NoPgo + # run coreclr perftiger microbenchmarks pgo perf jobs + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml + buildConfig: release + runtimeFlavor: coreclr + platforms: + - windows_x64 + jobParameters: + testGroup: perf + liveLibrariesBuildConfig: Release + projectFile: microbenchmarks.proj + runKind: micro + runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml + logicalmachine: 'perftiger' + pgoRunType: -NoPgo - #- template: /eng/pipelines/common/platform-matrix.yml - # parameters: - # jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml - # buildConfig: release - # runtimeFlavor: coreclr - # platforms: - # - windows_x64 - # jobParameters: - # testGroup: perf - # liveLibrariesBuildConfig: Release - # projectFile: microbenchmarks.proj - # runKind: micro - # runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml - # logicalmachine: 'perftiger' - # pgoRunType: -DynamicPgo + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml + buildConfig: release + runtimeFlavor: coreclr + platforms: + - windows_x64 + jobParameters: + testGroup: perf + liveLibrariesBuildConfig: Release + projectFile: microbenchmarks.proj + runKind: micro + runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml + logicalmachine: 'perftiger' + pgoRunType: -DynamicPgo - #- template: /eng/pipelines/common/platform-matrix.yml - # parameters: - # jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml - # buildConfig: release - # runtimeFlavor: coreclr - # platforms: - # - windows_x64 - # jobParameters: - # testGroup: perf - # liveLibrariesBuildConfig: Release - # projectFile: microbenchmarks.proj - # runKind: micro - # runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml - # logicalmachine: 'perftiger' - # pgoRunType: -FullPgo + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml + buildConfig: release + runtimeFlavor: coreclr + platforms: + - windows_x64 + jobParameters: + testGroup: perf + liveLibrariesBuildConfig: Release + projectFile: microbenchmarks.proj + runKind: micro + runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml + logicalmachine: 'perftiger' + pgoRunType: -FullPgo - ## run coreclr perfowl microbenchmarks perf job - #- template: /eng/pipelines/common/platform-matrix.yml - # parameters: - # jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml - # buildConfig: release - # runtimeFlavor: coreclr - # platforms: - # - Linux_x64 - # - windows_x64 - # jobParameters: - # testGroup: perf - # liveLibrariesBuildConfig: Release - # projectFile: microbenchmarks.proj - # runKind: micro - # runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml - # logicalmachine: 'perfowl' + # run coreclr perfowl microbenchmarks perf job + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml + buildConfig: release + runtimeFlavor: coreclr + platforms: + - Linux_x64 + - windows_x64 + jobParameters: + testGroup: perf + liveLibrariesBuildConfig: Release + projectFile: microbenchmarks.proj + runKind: micro + runJobTemplate: /eng/pipelines/coreclr/templates/run-performance-job.yml + logicalmachine: 'perfowl' - ## run coreclr crossgen perf job - #- template: /eng/pipelines/common/platform-matrix.yml - # parameters: - # jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml - # buildConfig: release - # runtimeFlavor: coreclr - # platforms: - # - Linux_x64 - # - windows_x64 - # - windows_x86 - # jobParameters: - # testGroup: perf - # liveLibrariesBuildConfig: Release - # projectFile: crossgen_perf.proj - # runKind: crossgen_scenarios - # runJobTemplate: /eng/pipelines/coreclr/templates/run-scenarios-job.yml - # logicalmachine: 'perftiger' + # run coreclr crossgen perf job + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml + buildConfig: release + runtimeFlavor: coreclr + platforms: + - Linux_x64 + - windows_x64 + - windows_x86 + jobParameters: + testGroup: perf + liveLibrariesBuildConfig: Release + projectFile: crossgen_perf.proj + runKind: crossgen_scenarios + runJobTemplate: /eng/pipelines/coreclr/templates/run-scenarios-job.yml + logicalmachine: 'perftiger' - ## run mono wasm blazor perf job - #- template: /eng/pipelines/common/platform-matrix.yml - # parameters: - # jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml - # buildConfig: release - # runtimeFlavor: wasm - # platforms: - # - Linux_x64 - # jobParameters: - # testGroup: perf - # liveLibrariesBuildConfig: Release - # runtimeType: wasm - # projectFile: blazor_perf.proj - # runKind: blazor_scenarios - # runJobTemplate: /eng/pipelines/coreclr/templates/run-scenarios-job.yml - # additionalSetupParameters: '--latestdotnet' - # logicalmachine: 'perftiger' + # run mono wasm blazor perf job + - template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/perf-job.yml + buildConfig: release + runtimeFlavor: wasm + platforms: + - Linux_x64 + jobParameters: + testGroup: perf + liveLibrariesBuildConfig: Release + runtimeType: wasm + projectFile: blazor_perf.proj + runKind: blazor_scenarios + runJobTemplate: /eng/pipelines/coreclr/templates/run-scenarios-job.yml + additionalSetupParameters: '--latestdotnet' + logicalmachine: 'perftiger' # build maui workload packs - template: /eng/pipelines/common/platform-matrix.yml diff --git a/eng/pipelines/coreclr/templates/build-perf-sample-apps.yml b/eng/pipelines/coreclr/templates/build-perf-sample-apps.yml index 6e8ace4e7935c2..57d99f231c5f82 100644 --- a/eng/pipelines/coreclr/templates/build-perf-sample-apps.yml +++ b/eng/pipelines/coreclr/templates/build-perf-sample-apps.yml @@ -35,19 +35,6 @@ steps: - script: rm -r -f $(Build.SourcesDirectory)/artifacts/bin/AndroidSampleApp workingDirectory: $(Build.SourcesDirectory)/artifacts/bin displayName: clean bindir - - script: make run MONO_ARCH=arm64 DEPLOY_AND_RUN=false RUNTIME_COMPONENTS=diagnostics_tracing - workingDirectory: $(Build.SourcesDirectory)/src/mono/sample/Android - displayName: Build HelloAndroid sample app - - script: mv $(Build.SourcesDirectory)/artifacts/bin/AndroidSampleApp/arm64/Release/android-arm64/publish/apk/bin/HelloAndroid.apk $(Build.SourcesDirectory)/artifacts/bin/AndroidSampleApp/arm64/Release/android-arm64/publish/apk/bin/HelloAndroidWithDiag.apk - - template: /eng/pipelines/common/upload-artifact-step.yml - parameters: - rootFolder: $(Build.SourcesDirectory)/artifacts/bin/AndroidSampleApp/arm64/Release/android-arm64/publish/apk/bin/HelloAndroidWithDiag.apk - includeRootFolder: true - displayName: Android Mono Artifacts With Diag - artifactName: AndroidMonoWithDiagarm64 - archiveExtension: '.tar.gz' - archiveType: tar - tarCompression: gz - ${{ if eq(parameters.osGroup, 'iOS') }}: - script: make build-appbundle TARGET=iOS MONO_ARCH=arm64 MONO_CONFIG=Release AOT=True USE_LLVM=False DEPLOY_AND_RUN=false diff --git a/eng/testing/performance/android_scenarios.proj b/eng/testing/performance/android_scenarios.proj index 841dc23687cd89..0b55442e304a2e 100644 --- a/eng/testing/performance/android_scenarios.proj +++ b/eng/testing/performance/android_scenarios.proj @@ -23,40 +23,40 @@ - + $(WorkItemDirectory) - echo on;set XHARNESSPATH=$(XharnessPath);cd $(ScenarioDirectory)helloandroid;copy %HELIX_CORRELATION_PAYLOAD%\HelloAndroid.apk .;copy %HELIX_CORRELATION_PAYLOAD%\HelloAndroidWithDiag.apk .;$(Python) pre.py - $(Python) -u test.py devicestartup --device-type android --package-path pub\HelloAndroidWithDiag.apk --package-name net.dot.HelloAndroid --exit-code 42 --scenario-name "%(Identity)" + echo on;set XHARNESSPATH=$(XharnessPath);cd $(ScenarioDirectory)helloandroid;copy %HELIX_CORRELATION_PAYLOAD%\HelloAndroid.apk .;$(Python) pre.py + $(Python) test.py devicestartup --device-type android --package-path pub\HelloAndroid.apk --package-name net.dot.HelloAndroid --scenario-name "%(Identity)" $(Python) post.py $(WorkItemDirectory) echo on;set XHARNESSPATH=$(XharnessPath);cd $(ScenarioDirectory)mauiandroid;copy %HELIX_CORRELATION_PAYLOAD%\MauiAndroidDefault.apk .;$(Python) pre.py - $(Python) -u test.py devicestartup --device-type android --package-path pub\MauiAndroidDefault.apk --package-name com.companyname.MauiTesting --exit-code 0 --scenario-name "%(Identity)" + $(Python) test.py devicestartup --device-type android --package-path pub\MauiAndroidDefault.apk --package-name com.companyname.MauiTesting --scenario-name "%(Identity)" $(Python) post.py diff --git a/eng/testing/performance/performance-setup.ps1 b/eng/testing/performance/performance-setup.ps1 index 72840befe2a03e..2e7c46d2fddb08 100644 --- a/eng/testing/performance/performance-setup.ps1 +++ b/eng/testing/performance/performance-setup.ps1 @@ -148,14 +148,9 @@ if ($AndroidMono) { { mkdir $WorkItemDirectory } - if(Test-Path "$SourceDirectory\androidHelloWorld\HelloAndroid.apk") { - Copy-Item -path "$SourceDirectory\androidHelloWorld\HelloAndroid.apk" $PayloadDirectory -Verbose - } - - if(Test-Path "$SourceDirectory\androidHelloWorldWithDiag\HelloAndroidWithDiag.apk") { - Copy-Item -path "$SourceDirectory\androidHelloWorldWithDiag\HelloAndroidWithDiag.apk" $PayloadDirectory -Verbose - } - Copy-Item -path "$SourceDirectory\MauiAndroidDefault.apk" $PayloadDirectory + + Copy-Item -path "$SourceDirectory\androidHelloWorld\HelloAndroid.apk" $PayloadDirectory -Verbose + Copy-Item -path "$SourceDirectory\MauiAndroidDefault.apk" $PayloadDirectory -Verbose $SetupArguments = $SetupArguments -replace $Architecture, 'arm64' } From 03380148e35c82efb807c2163907ba63be5a7363 Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Fri, 21 Jan 2022 14:02:35 -0800 Subject: [PATCH 07/14] Update the git clone to use the main repo again. --- eng/testing/performance/performance-setup.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/testing/performance/performance-setup.ps1 b/eng/testing/performance/performance-setup.ps1 index 2e7c46d2fddb08..7f1be6761b5049 100644 --- a/eng/testing/performance/performance-setup.ps1 +++ b/eng/testing/performance/performance-setup.ps1 @@ -124,7 +124,7 @@ if ($RunFromPerformanceRepo) { robocopy $SourceDirectory $PerformanceDirectory /E /XD $PayloadDirectory $SourceDirectory\artifacts $SourceDirectory\.git } else { - git clone --branch MinimalChangeStartupADB --depth 1 --quiet https://github.com/LoopedBard3/performance $PerformanceDirectory + git clone --branch main --depth 1 --quiet https://github.com/dotnet/performance $PerformanceDirectory } if($MonoDotnet -ne "") From 0c42bc2b13825912c2308b1d3709521a0041396d Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Fri, 21 Jan 2022 15:07:32 -0800 Subject: [PATCH 08/14] Missed the removal of the Diag download step. --- eng/pipelines/coreclr/templates/perf-job.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/eng/pipelines/coreclr/templates/perf-job.yml b/eng/pipelines/coreclr/templates/perf-job.yml index dd1ff0ebd3b242..ad55fef82e3189 100644 --- a/eng/pipelines/coreclr/templates/perf-job.yml +++ b/eng/pipelines/coreclr/templates/perf-job.yml @@ -166,13 +166,6 @@ jobs: artifactFileName: 'MauiAndroidApp.tar.gz' artifactName: 'MauiAndroidApp' displayName: 'Maui Android App' - - template: /eng/pipelines/common/download-artifact-step.yml - parameters: - unpackFolder: $(Build.SourcesDirectory)/androidHelloWorldWithDiag - cleanUnpackFolder: false - artifactFileName: 'AndroidMonoWithDiagarm64.tar.gz' - artifactName: 'AndroidMonoWithDiagarm64' - displayName: 'Mono Android Diagnostic Helloworld' # Download iOSMono tests and MauiiOS/MacCatalyst - ${{ if eq(parameters.runtimeType, 'iOSMono') }}: From 4b756a76bb4aea86b41f61c538237231e4219c0b Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Mon, 24 Jan 2022 08:54:26 -0800 Subject: [PATCH 09/14] Use test branch for full run test and keep it until the test finishes. --- eng/testing/performance/performance-setup.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/testing/performance/performance-setup.ps1 b/eng/testing/performance/performance-setup.ps1 index 7f1be6761b5049..2e7c46d2fddb08 100644 --- a/eng/testing/performance/performance-setup.ps1 +++ b/eng/testing/performance/performance-setup.ps1 @@ -124,7 +124,7 @@ if ($RunFromPerformanceRepo) { robocopy $SourceDirectory $PerformanceDirectory /E /XD $PayloadDirectory $SourceDirectory\artifacts $SourceDirectory\.git } else { - git clone --branch main --depth 1 --quiet https://github.com/dotnet/performance $PerformanceDirectory + git clone --branch MinimalChangeStartupADB --depth 1 --quiet https://github.com/LoopedBard3/performance $PerformanceDirectory } if($MonoDotnet -ne "") From 4404b383171d102b5d76e40d950569ab1aa63b07 Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Mon, 24 Jan 2022 08:55:48 -0800 Subject: [PATCH 10/14] Change perf pull branch back to main, hold until test run is complete. --- eng/testing/performance/performance-setup.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/testing/performance/performance-setup.ps1 b/eng/testing/performance/performance-setup.ps1 index 2e7c46d2fddb08..7f1be6761b5049 100644 --- a/eng/testing/performance/performance-setup.ps1 +++ b/eng/testing/performance/performance-setup.ps1 @@ -124,7 +124,7 @@ if ($RunFromPerformanceRepo) { robocopy $SourceDirectory $PerformanceDirectory /E /XD $PayloadDirectory $SourceDirectory\artifacts $SourceDirectory\.git } else { - git clone --branch MinimalChangeStartupADB --depth 1 --quiet https://github.com/LoopedBard3/performance $PerformanceDirectory + git clone --branch main --depth 1 --quiet https://github.com/dotnet/performance $PerformanceDirectory } if($MonoDotnet -ne "") From aff5e214f6102a8bfe5d56fc96c65961f4ba55de Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Fri, 28 Jan 2022 16:02:01 -0800 Subject: [PATCH 11/14] Add in iOS runtime builds for sure, and change branch for testing. --- eng/pipelines/coreclr/perf.yml | 2 ++ eng/testing/performance/performance-setup.ps1 | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/eng/pipelines/coreclr/perf.yml b/eng/pipelines/coreclr/perf.yml index ec233f348b0ecb..0655d07d3805d4 100644 --- a/eng/pipelines/coreclr/perf.yml +++ b/eng/pipelines/coreclr/perf.yml @@ -497,6 +497,8 @@ jobs: - Android_arm64 - MacCatalyst_x64 - iOSSimulator_x64 + - iOS_arm64 + - iOS_arm jobParameters: buildArgs: -s mono+libs+host+packs -c $(_BuildConfig) nameSuffix: Maui_Packs_Mono diff --git a/eng/testing/performance/performance-setup.ps1 b/eng/testing/performance/performance-setup.ps1 index 7f1be6761b5049..2e7c46d2fddb08 100644 --- a/eng/testing/performance/performance-setup.ps1 +++ b/eng/testing/performance/performance-setup.ps1 @@ -124,7 +124,7 @@ if ($RunFromPerformanceRepo) { robocopy $SourceDirectory $PerformanceDirectory /E /XD $PayloadDirectory $SourceDirectory\artifacts $SourceDirectory\.git } else { - git clone --branch main --depth 1 --quiet https://github.com/dotnet/performance $PerformanceDirectory + git clone --branch MinimalChangeStartupADB --depth 1 --quiet https://github.com/LoopedBard3/performance $PerformanceDirectory } if($MonoDotnet -ne "") From 514f5fffb397acaef69d90bcf2cc96943013f77f Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Fri, 28 Jan 2022 17:50:12 -0800 Subject: [PATCH 12/14] Add in the file extraction of the built ios packs and changed workload typo to runtime to better match actual meaning. --- eng/pipelines/coreclr/perf.yml | 2 +- eng/pipelines/coreclr/templates/build-perf-maui-apps.yml | 9 ++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/coreclr/perf.yml b/eng/pipelines/coreclr/perf.yml index 0655d07d3805d4..f71cb098841faf 100644 --- a/eng/pipelines/coreclr/perf.yml +++ b/eng/pipelines/coreclr/perf.yml @@ -484,7 +484,7 @@ jobs: additionalSetupParameters: '--latestdotnet' logicalmachine: 'perftiger' - # build maui workload packs + # build maui runtime packs - template: /eng/pipelines/common/platform-matrix.yml parameters: jobTemplate: /eng/pipelines/common/global-build-job.yml diff --git a/eng/pipelines/coreclr/templates/build-perf-maui-apps.yml b/eng/pipelines/coreclr/templates/build-perf-maui-apps.yml index 3940fb596a7368..2d80e85d77e804 100644 --- a/eng/pipelines/coreclr/templates/build-perf-maui-apps.yml +++ b/eng/pipelines/coreclr/templates/build-perf-maui-apps.yml @@ -31,7 +31,7 @@ steps: IntermediateArtifacts/MonoRuntimePacks/Shipping/Microsoft.NETCore.App.Runtime.Mono.maccatalyst-!(*.symbols).nupkg # Other artifacts to include once they are being built - # IntermediateArtifacts/MonoRuntimePacks/Shipping/Microsoft.NETCore.App.Runtime.Mono.maccatalyst-*.nupkg + # EX. IntermediateArtifacts/MonoRuntimePacks/Shipping/Microsoft.NETCore.App.Runtime.Mono.maccatalyst-*.nupkg - task: CopyFiles@2 displayName: Flatten packages @@ -80,6 +80,13 @@ steps: destinationFolder: $(Build.SourcesDirectory)/MauiTesting/ArtifactPacks/Microsoft.NETCore.App.Runtime.Mono.android-x64 overwriteExistingFiles: true cleanDestinationFolder: false + - task: ExtractFiles@1 + displayName: Extract ios-arm64 runtime + inputs: + archiveFilePatterns: $(Build.SourcesDirectory)/MauiTesting/ArtifactPacks/Microsoft.NETCore.App.Runtime.Mono.ios-arm64.*.zip + destinationFolder: $(Build.SourcesDirectory)/MauiTesting/ArtifactPacks/Microsoft.NETCore.App.Runtime.Mono.ios-arm64 + overwriteExistingFiles: true + cleanDestinationFolder: false - task: ExtractFiles@1 displayName: Extract maccatalyst-x64 runtime inputs: From 21e4337f6fb8f991cbac7b9563372a8a8eff6b68 Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Mon, 31 Jan 2022 08:55:01 -0800 Subject: [PATCH 13/14] Revert back to main branch for setup. --- eng/testing/performance/performance-setup.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/testing/performance/performance-setup.ps1 b/eng/testing/performance/performance-setup.ps1 index 2e7c46d2fddb08..7f1be6761b5049 100644 --- a/eng/testing/performance/performance-setup.ps1 +++ b/eng/testing/performance/performance-setup.ps1 @@ -124,7 +124,7 @@ if ($RunFromPerformanceRepo) { robocopy $SourceDirectory $PerformanceDirectory /E /XD $PayloadDirectory $SourceDirectory\artifacts $SourceDirectory\.git } else { - git clone --branch MinimalChangeStartupADB --depth 1 --quiet https://github.com/LoopedBard3/performance $PerformanceDirectory + git clone --branch main --depth 1 --quiet https://github.com/dotnet/performance $PerformanceDirectory } if($MonoDotnet -ne "") From 659de277d823e38bceb5f9365283ec240aa984e2 Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Mon, 31 Jan 2022 08:59:21 -0800 Subject: [PATCH 14/14] Git missed ios-arm runtime extraction, add that in. --- eng/pipelines/coreclr/templates/build-perf-maui-apps.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/eng/pipelines/coreclr/templates/build-perf-maui-apps.yml b/eng/pipelines/coreclr/templates/build-perf-maui-apps.yml index 2d80e85d77e804..f6d03413a1f52d 100644 --- a/eng/pipelines/coreclr/templates/build-perf-maui-apps.yml +++ b/eng/pipelines/coreclr/templates/build-perf-maui-apps.yml @@ -80,6 +80,13 @@ steps: destinationFolder: $(Build.SourcesDirectory)/MauiTesting/ArtifactPacks/Microsoft.NETCore.App.Runtime.Mono.android-x64 overwriteExistingFiles: true cleanDestinationFolder: false + - task: ExtractFiles@1 + displayName: Extract ios-arm runtime + inputs: + archiveFilePatterns: $(Build.SourcesDirectory)/MauiTesting/ArtifactPacks/Microsoft.NETCore.App.Runtime.Mono.ios-arm.*.zip + destinationFolder: $(Build.SourcesDirectory)/MauiTesting/ArtifactPacks/Microsoft.NETCore.App.Runtime.Mono.ios-arm + overwriteExistingFiles: true + cleanDestinationFolder: false - task: ExtractFiles@1 displayName: Extract ios-arm64 runtime inputs: