From 478571ca82dedc4f07f6a176709224adf3ee367a Mon Sep 17 00:00:00 2001 From: David Wrighton Date: Wed, 19 May 2021 18:02:06 -0700 Subject: [PATCH] Fix crossgen2 comp jobs (#52949) - Remove experimental utf8 dll from list of dlls to compile - Fix CORE_ROOT value to point at correct directory - Add testResults.xml support for better test diagnostics - Print out stdout and stderr for better debugging build time failures --- .../crossgen2-comparison-build-job.yml | 2 + .../templates/crossgen2-comparison-job.yml | 17 ++- .../Common/scripts/crossgen2_comparison.py | 138 ++++++++++++++++-- 3 files changed, 143 insertions(+), 14 deletions(-) diff --git a/eng/pipelines/coreclr/templates/crossgen2-comparison-build-job.yml b/eng/pipelines/coreclr/templates/crossgen2-comparison-build-job.yml index 1a326c1f295fc8..a595173f708b51 100644 --- a/eng/pipelines/coreclr/templates/crossgen2-comparison-build-job.yml +++ b/eng/pipelines/coreclr/templates/crossgen2-comparison-build-job.yml @@ -156,6 +156,7 @@ jobs: --result_dir $(workItemDirectory)/log --target_os $(target_crossgen2_os) --target_arch $(archType) + --compiler_arch_os $(target_crossgen2_os)_x64 ${{ if eq(parameters.osGroup, 'windows') }}: arguments: crossgen_framework @@ -165,6 +166,7 @@ jobs: --result_dir $(workItemDirectory)\log --target_os $(target_crossgen2_os) --target_arch $(archType) + --compiler_arch_os $(target_crossgen2_os)_x64 - task: PublishPipelineArtifact@1 displayName: Publish cross compiled component diff --git a/eng/pipelines/coreclr/templates/crossgen2-comparison-job.yml b/eng/pipelines/coreclr/templates/crossgen2-comparison-job.yml index 8bc7f6f0f6f06b..b3d04f99f4e3da 100644 --- a/eng/pipelines/coreclr/templates/crossgen2-comparison-job.yml +++ b/eng/pipelines/coreclr/templates/crossgen2-comparison-job.yml @@ -67,6 +67,8 @@ jobs: value: $(artifactsDirectory)$(dir)tests$(dir)coreclr$(dir)$(targetFlavor)$(dir)Tests$(dir)Core_Root - name: targetarch value: ${{ parameters.targetarch }} + - name: compiler_arch_os + value: ${{ format('{0}{1}_{2}', parameters.osGroup, parameters.osSubgroup, parameters.archType) }} - name: crossgencompare_build_artifact value: crossgen_comparison_build_${{ parameters.targetos }}_${{ parameters.targetarch }} - name: displayname_comparison_job @@ -146,34 +148,39 @@ jobs: echo Targeting $(targetFlavor) ; chmod +x $HELIX_WORKITEM_PAYLOAD/corerun; mkdir -p $HELIX_WORKITEM_PAYLOAD/log; - export CORE_ROOT=$HELIX_CORRELATION_PAYLOAD; + export CORE_ROOT=$HELIX_WORKITEM_PAYLOAD; python3 -u $HELIX_CORRELATION_PAYLOAD/crossgen2_comparison.py crossgen_framework --crossgen $HELIX_WORKITEM_PAYLOAD/crossgen2/crossgen2.dll --dotnet $HELIX_WORKITEM_PAYLOAD/corerun --core_root $HELIX_WORKITEM_PAYLOAD/prebuiltWork/dlls --result_dir $HELIX_WORKITEM_PAYLOAD/log --target_os $(target_crossgen2_os) - --target_arch $(targetarch); + --target_arch $(targetarch) + --compiler_arch_os $(compiler_arch_os); python3 -u $HELIX_CORRELATION_PAYLOAD/crossgen2_comparison.py compare --base_dir $HELIX_WORKITEM_PAYLOAD/prebuiltWork/log --diff_dir $HELIX_WORKITEM_PAYLOAD/log + --testresults $HELIX_WORKITEM_ROOT/testResults.xml + --target_arch_os $(target_crossgen2_os)_$(targetarch) ${{ if eq(parameters.osGroup, 'windows') }}: WorkItemCommand: echo $(displayname_comparison_job) & echo Targeting $(targetFlavor) & md %HELIX_WORKITEM_PAYLOAD%\log & - set CORE_ROOT=%HELIX_CORRELATION_PAYLOAD% & + set CORE_ROOT=%HELIX_WORKITEM_PAYLOAD%& python -u %HELIX_CORRELATION_PAYLOAD%\crossgen2_comparison.py crossgen_framework --crossgen %HELIX_WORKITEM_PAYLOAD%\crossgen2\crossgen2.dll --dotnet %HELIX_WORKITEM_PAYLOAD%\corerun.exe --core_root %HELIX_WORKITEM_PAYLOAD%\prebuiltWork\dlls --result_dir %HELIX_WORKITEM_PAYLOAD%\log --target_os $(target_crossgen2_os) - --target_arch $(targetarch) & + --target_arch $(targetarch) + --compiler_arch_os $(compiler_arch_os) & python -u %HELIX_CORRELATION_PAYLOAD%\crossgen2_comparison.py compare --base_dir %HELIX_WORKITEM_PAYLOAD%\prebuiltWork\log --diff_dir %HELIX_WORKITEM_PAYLOAD%\log - + --testresults %HELIX_WORKITEM_ROOT%\testResults.xml + --target_arch_os $(target_crossgen2_os)_$(targetarch) # Publish log - task: PublishPipelineArtifact@1 displayName: Publish log diff --git a/src/tests/Common/scripts/crossgen2_comparison.py b/src/tests/Common/scripts/crossgen2_comparison.py index fbd5261f2e897b..e9d751081c6319 100644 --- a/src/tests/Common/scripts/crossgen2_comparison.py +++ b/src/tests/Common/scripts/crossgen2_comparison.py @@ -104,6 +104,7 @@ import shutil import subprocess import sys +from xml.dom import minidom ################################################################################ # Argument Parser @@ -136,6 +137,7 @@ def build_argument_parser(): framework_parser.add_argument('--target_arch', dest='target_arch', required=True) framework_parser.add_argument('--core_root', dest='core_root', required=True) framework_parser.add_argument('--result_dir', dest='result_dirname', required=True) + framework_parser.add_argument('--compiler_arch_os', dest='compiler_arch_os', required=True) framework_parser.set_defaults(func=crossgen_framework) @@ -152,6 +154,8 @@ def build_argument_parser(): compare_parser = subparsers.add_parser('compare', description=compare_parser_description) compare_parser.add_argument('--base_dir', dest='base_dirname', required=True) compare_parser.add_argument('--diff_dir', dest='diff_dirname', required=True) + compare_parser.add_argument('--testresults', dest='testresultsxml', required=True) + compare_parser.add_argument('--target_arch_os', dest='target_arch_os', required=True) compare_parser.set_defaults(func=compare_results) return parser @@ -480,7 +484,6 @@ def run_to_completion(self, async_callback, *extra_args): 'System.Threading.Timer.dll', 'System.Transactions.dll', 'System.Transactions.Local.dll', - 'System.Utf8String.Experimental.dll', 'System.ValueTuple.dll', 'System.Web.dll', 'System.Web.HttpUtility.dll', @@ -586,7 +589,7 @@ def compute_file_hashsum(filename): # This describes collected during crossgen information. ################################################################################ class CrossGenResult: - def __init__(self, assembly_name, returncode, stdout, stderr, output_file_hashsum, output_file_size_in_bytes, output_file_type, args): + def __init__(self, assembly_name, returncode, stdout, stderr, output_file_hashsum, output_file_size_in_bytes, output_file_type, args, compiler_arch_os): self.assembly_name = assembly_name self.returncode = returncode self.stdout = stdout @@ -595,6 +598,7 @@ def __init__(self, assembly_name, returncode, stdout, stderr, output_file_hashsu self.output_file_size_in_bytes = output_file_size_in_bytes self.output_file_type = output_file_type self.args = args + self.compiler_arch_os = compiler_arch_os ################################################################################ # JSON Encoder for CrossGenResult objects. @@ -603,10 +607,11 @@ class CrossGenResultEncoder(json.JSONEncoder): def default(self, obj): if isinstance(obj, CrossGenResult): return { + 'CompilerArchOS': obj.compiler_arch_os, 'AssemblyName': obj.assembly_name, 'ReturnCode': obj.returncode, - 'StdOut': obj.stdout.splitlines(), - 'StdErr': obj.stderr.splitlines(), + 'StdOut': obj.stdout if isinstance(obj.stdout, list) else obj.stdout.splitlines(), + 'StdErr': obj.stderr if isinstance(obj.stderr, list) else obj.stderr.splitlines(), 'OutputFileHash': obj.output_file_hashsum, 'OutputFileSizeInBytes': obj.output_file_size_in_bytes, 'OutputFileType': obj.output_file_type } @@ -621,6 +626,7 @@ def __init__(self, *args, **kwargs): json.JSONDecoder.__init__(self, object_hook=self._decode_object, *args, **kwargs) def _decode_object(self, dict): try: + compiler_arch_os = dict['CompilerArchOS'] assembly_name = dict['AssemblyName'] returncode = dict['ReturnCode'] stdout = dict['StdOut'] @@ -628,7 +634,7 @@ def _decode_object(self, dict): output_file_hashsum = dict['OutputFileHash'] output_file_size_in_bytes = dict['OutputFileSizeInBytes'] output_file_type = dict['OutputFileType'] - return CrossGenResult(assembly_name, returncode, stdout, stderr, output_file_hashsum, output_file_size_in_bytes, output_file_type, "") + return CrossGenResult(assembly_name, returncode, stdout, stderr, output_file_hashsum, output_file_size_in_bytes, output_file_type, "", compiler_arch_os) except KeyError: return dict @@ -645,13 +651,13 @@ class FileTypes: NativeOrReadyToRunImage = 'NativeOrReadyToRunImage' DebuggingFile = 'DebuggingFile' -async def run_crossgen(dotnet, crossgen_executable_filename, il_filename, ni_filename, platform_assemblies_paths, debugging_files_dirname, target_os, target_arch): +async def run_crossgen(dotnet, crossgen_executable_filename, il_filename, ni_filename, platform_assemblies_paths, debugging_files_dirname, target_os, target_arch, compiler_arch_os): runner = CrossGenRunner(dotnet, crossgen_executable_filename) returncode, stdout, stderr, args = await runner.crossgen_il_file(il_filename, ni_filename, platform_assemblies_paths, target_os, target_arch) ni_file_hashsum = compute_file_hashsum(ni_filename) if returncode == 0 else None ni_file_size_in_bytes = os.path.getsize(ni_filename) if returncode == 0 else None assembly_name = get_assembly_name(il_filename) - crossgen_assembly_result = CrossGenResult(assembly_name, returncode, stdout, stderr, ni_file_hashsum, ni_file_size_in_bytes, output_file_type=FileTypes.NativeOrReadyToRunImage, args=args) + crossgen_assembly_result = CrossGenResult(assembly_name, returncode, stdout, stderr, ni_file_hashsum, ni_file_size_in_bytes, output_file_type=FileTypes.NativeOrReadyToRunImage, args=args, compiler_arch_os=compiler_arch_os) if returncode != 0: return [crossgen_assembly_result] @@ -722,10 +728,10 @@ async def run_crossgen_helper(print_prefix, assembly_name): il_filename = os.path.join(args.core_root, assembly_name) ni_filename = os.path.join(ni_files_dirname, add_ni_extension(assembly_name)) - crossgen_results = await run_crossgen(args.dotnet, args.crossgen_executable_filename, il_filename, ni_filename, platform_assemblies_paths, debugging_files_dirname, args.target_os, args.target_arch) + crossgen_results = await run_crossgen(args.dotnet, args.crossgen_executable_filename, il_filename, ni_filename, platform_assemblies_paths, debugging_files_dirname, args.target_os, args.target_arch, args.compiler_arch_os) if crossgen_results[0].returncode != 0: g_frameworkcompile_failed = True - print("{}{} {} return code={} args'{}'".format(print_prefix, args.crossgen_executable_filename, assembly_name, crossgen_results[0].returncode, crossgen_results[0].args)) + print("{}{} {} return code={} args'{}' stdout '{}' stderr'{}'".format(print_prefix, args.crossgen_executable_filename, assembly_name, crossgen_results[0].returncode, crossgen_results[0].args, crossgen_results[0].stdout, crossgen_results[0].stderr)) save_crossgen_results_to_json_files(crossgen_results, args.result_dirname) helper = AsyncSubprocessHelper(g_Framework_Assemblies, verbose=True) @@ -868,6 +874,120 @@ def compare_results(args): print("Number of mismatched results: {0}".format(num_mismatched_results)) print("Total number of files compared: {0}".format(len(both_assemblies))) + root = minidom.Document() + assemblies = root.createElement('assemblies') + root.appendChild(assemblies) + + assembly = root.createElement('assembly') + assembly.setAttribute('name', 'crossgen2_comparison_job_targetting_{0}'.format(args.target_arch_os)) + assembly.setAttribute('total', '{0}'.format(len(both_assemblies))) + assembly.setAttribute('passed', '{0}'.format(len(both_assemblies) - num_omitted_results - num_mismatched_results)) + assembly.setAttribute('failed', '{0}'.format(num_omitted_results+num_mismatched_results)) + assembly.setAttribute('skipped', '0') + assemblies.appendChild(assembly) + + collection = root.createElement('collection') + collection.setAttribute('name', 'crossgen2_comparison_job_targetting_{0}'.format(args.target_arch_os)) + collection.setAttribute('total', '{0}'.format(len(both_assemblies))) + collection.setAttribute('passed', '{0}'.format(len(both_assemblies) - num_omitted_results - num_mismatched_results)) + collection.setAttribute('failed', '{0}'.format(num_omitted_results+num_mismatched_results)) + collection.setAttribute('skipped', '0') + assembly.appendChild(collection) + + for assembly_name in sorted(omitted_from_base_dir): + diff_result = diff_results_by_name[assembly_name] + message = 'Expected nothing, got {0}'.format(json.dumps(diff_result, cls=CrossGenResultEncoder, indent=2)) + testresult = root.createElement('test') + testresult.setAttribute('name', 'CrossgenCompile_{2}_Target_{0}_Omitted_vs_{1}'.format(args.target_arch_os, diff_result.compiler_arch_os, assembly_name)) + testresult.setAttribute('type', 'Target_{0}'.format(args.target_arch_os)) + testresult.setAttribute('method', diff_result.compiler_arch_os) + testresult.setAttribute('time', '0') + testresult.setAttribute('result', 'Fail') + collection.appendChild(testresult) + + failureXml = root.createElement('failure') + failureXml.setAttribute('exception-type', 'OmittedFromBase') + testresult.appendChild(failureXml) + + messageXml = root.createElement('message') + messageXml.appendChild(root.createTextNode(message)) + failureXml.appendChild(messageXml) + messageXml = root.createElement('output') + messageXml.appendChild(root.createTextNode(message)) + failureXml.appendChild(messageXml) + + for assembly_name in omitted_from_diff_dir: + base_result = diff_results_by_name[assembly_name] + message = 'Expected {0} got nothing'.format(json.dumps(base_result, cls=CrossGenResultEncoder, indent=2)) + testresult = root.createElement('test') + testresult.setAttribute('name', 'CrossgenCompile_{2}_Target_{0}_{1}_vs__Omitted'.format(args.target_arch_os, base_result.compiler_arch_os, assembly_name)) + testresult.setAttribute('type', 'Target_{0}'.format(args.target_arch_os)) + testresult.setAttribute('method', base_result.compiler_arch_os) + testresult.setAttribute('time', '0') + testresult.setAttribute('result', 'Fail') + collection.appendChild(testresult) + + failureXml = root.createElement('failure') + failureXml.setAttribute('exception-type', 'OmittedFromDiff') + testresult.appendChild(failureXml) + + messageXml = root.createElement('message') + messageXml.appendChild(root.createTextNode(message)) + failureXml.appendChild(messageXml) + messageXml = root.createElement('output') + messageXml.appendChild(root.createTextNode(message)) + failureXml.appendChild(messageXml) + + for assembly_name in sorted(both_assemblies): + base_result = base_results_by_name[assembly_name] + diff_result = diff_results_by_name[assembly_name] + base_diff_are_equal = True + + if base_result.returncode != diff_result.returncode: + base_diff_are_equal = False + elif base_result.returncode == 0 and diff_result.returncode == 0: + if base_result.output_file_hashsum != diff_result.output_file_hashsum: + base_diff_are_equal = False + if base_result.output_file_size_in_bytes != diff_result.output_file_size_in_bytes: + base_diff_are_equal = False + else: + base_diff_are_equal = False + + base_result_string = json.dumps(base_result, cls=CrossGenResultEncoder, indent=2) + diff_result_string = json.dumps(diff_result, cls=CrossGenResultEncoder, indent=2) + message = 'Expected {0} got {1}'.format(base_result_string, diff_result_string) + testresult = root.createElement('test') + testresult.setAttribute('name', 'CrossgenCompile_{3}_Target_{0}_{1}_vs_{2}'.format(args.target_arch_os, base_result.compiler_arch_os, diff_result.compiler_arch_os, assembly_name)) + testresult.setAttribute('type', 'Target_{0}'.format(args.target_arch_os)) + testresult.setAttribute('method', '{0}_{1}'.format(base_result.compiler_arch_os, diff_result.compiler_arch_os)) + testresult.setAttribute('time', '0') + if base_diff_are_equal: + testresult.setAttribute('result', 'Pass') + else: + testresult.setAttribute('result', 'Fail') + + collection.appendChild(testresult) + + if not base_diff_are_equal: + failureXml = root.createElement('failure') + failureXml.setAttribute('exception-type', 'MismatchOrReturnCodeFail') + testresult.appendChild(failureXml) + + messageXml = root.createElement('message') + messageXml.appendChild(root.createTextNode(message)) + + failureXml.appendChild(messageXml) + messageXml = root.createElement('output') + messageXml.appendChild(root.createTextNode(message)) + + failureXml.appendChild(messageXml) + + xml_str = root.toprettyxml(indent ="\t") + + if output_file_type == FileTypes.NativeOrReadyToRunImage: + with open(args.testresultsxml, "w") as f: + f.write(xml_str) + if not did_compare: sys.exit(1)