Skip to content

Commit

Permalink
Fix crossgen2 comp jobs (#52949)
Browse files Browse the repository at this point in the history 
- Remove experimental utf8 dll from list of dlls to compile
- Fix CORE_ROOT value to point at correct directory
- Add testResults.xml support for better test diagnostics
- Print out stdout and stderr for better debugging build time failures
  • Loading branch information
@davidwrighton
davidwrighton authored May 20, 2021
1 parent 84233f1 commit 478571c
Show file tree
Hide file tree
Showing 3 changed files with 143 additions and 14 deletions.
2 changes: 2 additions & 0 deletions eng/pipelines/coreclr/templates/crossgen2-comparison-build-job.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,7 @@ jobs:
--result_dir $(workItemDirectory)/log
--target_os $(target_crossgen2_os)
--target_arch $(archType)
--compiler_arch_os $(target_crossgen2_os)_x64
${{ if eq(parameters.osGroup, 'windows') }}:
arguments:
crossgen_framework
Expand All @@ -165,6 +166,7 @@ jobs:
--result_dir $(workItemDirectory)\log
--target_os $(target_crossgen2_os)
--target_arch $(archType)
--compiler_arch_os $(target_crossgen2_os)_x64

- task: PublishPipelineArtifact@1
displayName: Publish cross compiled component
Expand Down
17 changes: 12 additions & 5 deletions eng/pipelines/coreclr/templates/crossgen2-comparison-job.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,8 @@ jobs:
value: $(artifactsDirectory)$(dir)tests$(dir)coreclr$(dir)$(targetFlavor)$(dir)Tests$(dir)Core_Root
- name: targetarch
value: ${{ parameters.targetarch }}
- name: compiler_arch_os
value: ${{ format('{0}{1}_{2}', parameters.osGroup, parameters.osSubgroup, parameters.archType) }}
- name: crossgencompare_build_artifact
value: crossgen_comparison_build_${{ parameters.targetos }}_${{ parameters.targetarch }}
- name: displayname_comparison_job
Expand Down Expand Up @@ -146,34 +148,39 @@ jobs:
echo Targeting $(targetFlavor) ;
chmod +x $HELIX_WORKITEM_PAYLOAD/corerun;
mkdir -p $HELIX_WORKITEM_PAYLOAD/log;
export CORE_ROOT=$HELIX_CORRELATION_PAYLOAD;
export CORE_ROOT=$HELIX_WORKITEM_PAYLOAD;
python3 -u $HELIX_CORRELATION_PAYLOAD/crossgen2_comparison.py crossgen_framework
--crossgen $HELIX_WORKITEM_PAYLOAD/crossgen2/crossgen2.dll
--dotnet $HELIX_WORKITEM_PAYLOAD/corerun
--core_root $HELIX_WORKITEM_PAYLOAD/prebuiltWork/dlls
--result_dir $HELIX_WORKITEM_PAYLOAD/log
--target_os $(target_crossgen2_os)
--target_arch $(targetarch);
--target_arch $(targetarch)
--compiler_arch_os $(compiler_arch_os);
python3 -u $HELIX_CORRELATION_PAYLOAD/crossgen2_comparison.py compare
--base_dir $HELIX_WORKITEM_PAYLOAD/prebuiltWork/log
--diff_dir $HELIX_WORKITEM_PAYLOAD/log
--testresults $HELIX_WORKITEM_ROOT/testResults.xml
--target_arch_os $(target_crossgen2_os)_$(targetarch)
${{ if eq(parameters.osGroup, 'windows') }}:
WorkItemCommand:
echo $(displayname_comparison_job) &
echo Targeting $(targetFlavor) &
md %HELIX_WORKITEM_PAYLOAD%\log &
set CORE_ROOT=%HELIX_CORRELATION_PAYLOAD% &
set CORE_ROOT=%HELIX_WORKITEM_PAYLOAD%&
python -u %HELIX_CORRELATION_PAYLOAD%\crossgen2_comparison.py crossgen_framework
--crossgen %HELIX_WORKITEM_PAYLOAD%\crossgen2\crossgen2.dll
--dotnet %HELIX_WORKITEM_PAYLOAD%\corerun.exe
--core_root %HELIX_WORKITEM_PAYLOAD%\prebuiltWork\dlls
--result_dir %HELIX_WORKITEM_PAYLOAD%\log
--target_os $(target_crossgen2_os)
--target_arch $(targetarch) &
--target_arch $(targetarch)
--compiler_arch_os $(compiler_arch_os) &
python -u %HELIX_CORRELATION_PAYLOAD%\crossgen2_comparison.py compare
--base_dir %HELIX_WORKITEM_PAYLOAD%\prebuiltWork\log
--diff_dir %HELIX_WORKITEM_PAYLOAD%\log

--testresults %HELIX_WORKITEM_ROOT%\testResults.xml
--target_arch_os $(target_crossgen2_os)_$(targetarch)
# Publish log
- task: PublishPipelineArtifact@1
displayName: Publish log
Expand Down
138 changes: 129 additions & 9 deletions src/tests/Common/scripts/crossgen2_comparison.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,7 @@
import shutil
import subprocess
import sys
from xml.dom import minidom

################################################################################
# Argument Parser
Expand Down Expand Up @@ -136,6 +137,7 @@ def build_argument_parser():
framework_parser.add_argument('--target_arch', dest='target_arch', required=True)
framework_parser.add_argument('--core_root', dest='core_root', required=True)
framework_parser.add_argument('--result_dir', dest='result_dirname', required=True)
framework_parser.add_argument('--compiler_arch_os', dest='compiler_arch_os', required=True)
framework_parser.set_defaults(func=crossgen_framework)


Expand All @@ -152,6 +154,8 @@ def build_argument_parser():
compare_parser = subparsers.add_parser('compare', description=compare_parser_description)
compare_parser.add_argument('--base_dir', dest='base_dirname', required=True)
compare_parser.add_argument('--diff_dir', dest='diff_dirname', required=True)
compare_parser.add_argument('--testresults', dest='testresultsxml', required=True)
compare_parser.add_argument('--target_arch_os', dest='target_arch_os', required=True)
compare_parser.set_defaults(func=compare_results)

return parser
Expand Down Expand Up @@ -480,7 +484,6 @@ def run_to_completion(self, async_callback, *extra_args):
'System.Threading.Timer.dll',
'System.Transactions.dll',
'System.Transactions.Local.dll',
'System.Utf8String.Experimental.dll',
'System.ValueTuple.dll',
'System.Web.dll',
'System.Web.HttpUtility.dll',
Expand Down Expand Up @@ -586,7 +589,7 @@ def compute_file_hashsum(filename):
# This describes collected during crossgen information.
################################################################################
class CrossGenResult:
def __init__(self, assembly_name, returncode, stdout, stderr, output_file_hashsum, output_file_size_in_bytes, output_file_type, args):
def __init__(self, assembly_name, returncode, stdout, stderr, output_file_hashsum, output_file_size_in_bytes, output_file_type, args, compiler_arch_os):
self.assembly_name = assembly_name
self.returncode = returncode
self.stdout = stdout
Expand All @@ -595,6 +598,7 @@ def __init__(self, assembly_name, returncode, stdout, stderr, output_file_hashsu
self.output_file_size_in_bytes = output_file_size_in_bytes
self.output_file_type = output_file_type
self.args = args
self.compiler_arch_os = compiler_arch_os

################################################################################
# JSON Encoder for CrossGenResult objects.
Expand All @@ -603,10 +607,11 @@ class CrossGenResultEncoder(json.JSONEncoder):
def default(self, obj):
if isinstance(obj, CrossGenResult):
return {
'CompilerArchOS': obj.compiler_arch_os,
'AssemblyName': obj.assembly_name,
'ReturnCode': obj.returncode,
'StdOut': obj.stdout.splitlines(),
'StdErr': obj.stderr.splitlines(),
'StdOut': obj.stdout if isinstance(obj.stdout, list) else obj.stdout.splitlines(),
'StdErr': obj.stderr if isinstance(obj.stderr, list) else obj.stderr.splitlines(),
'OutputFileHash': obj.output_file_hashsum,
'OutputFileSizeInBytes': obj.output_file_size_in_bytes,
'OutputFileType': obj.output_file_type }
Expand All @@ -621,14 +626,15 @@ def __init__(self, *args, **kwargs):
json.JSONDecoder.__init__(self, object_hook=self._decode_object, *args, **kwargs)
def _decode_object(self, dict):
try:
compiler_arch_os = dict['CompilerArchOS']
assembly_name = dict['AssemblyName']
returncode = dict['ReturnCode']
stdout = dict['StdOut']
stderr = dict['StdErr']
output_file_hashsum = dict['OutputFileHash']
output_file_size_in_bytes = dict['OutputFileSizeInBytes']
output_file_type = dict['OutputFileType']
return CrossGenResult(assembly_name, returncode, stdout, stderr, output_file_hashsum, output_file_size_in_bytes, output_file_type, "")
return CrossGenResult(assembly_name, returncode, stdout, stderr, output_file_hashsum, output_file_size_in_bytes, output_file_type, "", compiler_arch_os)
except KeyError:
return dict

Expand All @@ -645,13 +651,13 @@ class FileTypes:
NativeOrReadyToRunImage = 'NativeOrReadyToRunImage'
DebuggingFile = 'DebuggingFile'

async def run_crossgen(dotnet, crossgen_executable_filename, il_filename, ni_filename, platform_assemblies_paths, debugging_files_dirname, target_os, target_arch):
async def run_crossgen(dotnet, crossgen_executable_filename, il_filename, ni_filename, platform_assemblies_paths, debugging_files_dirname, target_os, target_arch, compiler_arch_os):
runner = CrossGenRunner(dotnet, crossgen_executable_filename)
returncode, stdout, stderr, args = await runner.crossgen_il_file(il_filename, ni_filename, platform_assemblies_paths, target_os, target_arch)
ni_file_hashsum = compute_file_hashsum(ni_filename) if returncode == 0 else None
ni_file_size_in_bytes = os.path.getsize(ni_filename) if returncode == 0 else None
assembly_name = get_assembly_name(il_filename)
crossgen_assembly_result = CrossGenResult(assembly_name, returncode, stdout, stderr, ni_file_hashsum, ni_file_size_in_bytes, output_file_type=FileTypes.NativeOrReadyToRunImage, args=args)
crossgen_assembly_result = CrossGenResult(assembly_name, returncode, stdout, stderr, ni_file_hashsum, ni_file_size_in_bytes, output_file_type=FileTypes.NativeOrReadyToRunImage, args=args, compiler_arch_os=compiler_arch_os)

if returncode != 0:
return [crossgen_assembly_result]
Expand Down Expand Up @@ -722,10 +728,10 @@ async def run_crossgen_helper(print_prefix, assembly_name):

il_filename = os.path.join(args.core_root, assembly_name)
ni_filename = os.path.join(ni_files_dirname, add_ni_extension(assembly_name))
crossgen_results = await run_crossgen(args.dotnet, args.crossgen_executable_filename, il_filename, ni_filename, platform_assemblies_paths, debugging_files_dirname, args.target_os, args.target_arch)
crossgen_results = await run_crossgen(args.dotnet, args.crossgen_executable_filename, il_filename, ni_filename, platform_assemblies_paths, debugging_files_dirname, args.target_os, args.target_arch, args.compiler_arch_os)
if crossgen_results[0].returncode != 0:
g_frameworkcompile_failed = True
print("{}{} {} return code={} args'{}'".format(print_prefix, args.crossgen_executable_filename, assembly_name, crossgen_results[0].returncode, crossgen_results[0].args))
print("{}{} {} return code={} args'{}' stdout '{}' stderr'{}'".format(print_prefix, args.crossgen_executable_filename, assembly_name, crossgen_results[0].returncode, crossgen_results[0].args, crossgen_results[0].stdout, crossgen_results[0].stderr))
save_crossgen_results_to_json_files(crossgen_results, args.result_dirname)

helper = AsyncSubprocessHelper(g_Framework_Assemblies, verbose=True)
Expand Down Expand Up @@ -868,6 +874,120 @@ def compare_results(args):
print("Number of mismatched results: {0}".format(num_mismatched_results))
print("Total number of files compared: {0}".format(len(both_assemblies)))

root = minidom.Document()
assemblies = root.createElement('assemblies')
root.appendChild(assemblies)

assembly = root.createElement('assembly')
assembly.setAttribute('name', 'crossgen2_comparison_job_targetting_{0}'.format(args.target_arch_os))
assembly.setAttribute('total', '{0}'.format(len(both_assemblies)))
assembly.setAttribute('passed', '{0}'.format(len(both_assemblies) - num_omitted_results - num_mismatched_results))
assembly.setAttribute('failed', '{0}'.format(num_omitted_results+num_mismatched_results))
assembly.setAttribute('skipped', '0')
assemblies.appendChild(assembly)

collection = root.createElement('collection')
collection.setAttribute('name', 'crossgen2_comparison_job_targetting_{0}'.format(args.target_arch_os))
collection.setAttribute('total', '{0}'.format(len(both_assemblies)))
collection.setAttribute('passed', '{0}'.format(len(both_assemblies) - num_omitted_results - num_mismatched_results))
collection.setAttribute('failed', '{0}'.format(num_omitted_results+num_mismatched_results))
collection.setAttribute('skipped', '0')
assembly.appendChild(collection)

for assembly_name in sorted(omitted_from_base_dir):
diff_result = diff_results_by_name[assembly_name]
message = 'Expected nothing, got {0}'.format(json.dumps(diff_result, cls=CrossGenResultEncoder, indent=2))
testresult = root.createElement('test')
testresult.setAttribute('name', 'CrossgenCompile_{2}_Target_{0}_Omitted_vs_{1}'.format(args.target_arch_os, diff_result.compiler_arch_os, assembly_name))
testresult.setAttribute('type', 'Target_{0}'.format(args.target_arch_os))
testresult.setAttribute('method', diff_result.compiler_arch_os)
testresult.setAttribute('time', '0')
testresult.setAttribute('result', 'Fail')
collection.appendChild(testresult)

failureXml = root.createElement('failure')
failureXml.setAttribute('exception-type', 'OmittedFromBase')
testresult.appendChild(failureXml)

messageXml = root.createElement('message')
messageXml.appendChild(root.createTextNode(message))
failureXml.appendChild(messageXml)
messageXml = root.createElement('output')
messageXml.appendChild(root.createTextNode(message))
failureXml.appendChild(messageXml)

for assembly_name in omitted_from_diff_dir:
base_result = diff_results_by_name[assembly_name]
message = 'Expected {0} got nothing'.format(json.dumps(base_result, cls=CrossGenResultEncoder, indent=2))
testresult = root.createElement('test')
testresult.setAttribute('name', 'CrossgenCompile_{2}_Target_{0}_{1}_vs__Omitted'.format(args.target_arch_os, base_result.compiler_arch_os, assembly_name))
testresult.setAttribute('type', 'Target_{0}'.format(args.target_arch_os))
testresult.setAttribute('method', base_result.compiler_arch_os)
testresult.setAttribute('time', '0')
testresult.setAttribute('result', 'Fail')
collection.appendChild(testresult)

failureXml = root.createElement('failure')
failureXml.setAttribute('exception-type', 'OmittedFromDiff')
testresult.appendChild(failureXml)

messageXml = root.createElement('message')
messageXml.appendChild(root.createTextNode(message))
failureXml.appendChild(messageXml)
messageXml = root.createElement('output')
messageXml.appendChild(root.createTextNode(message))
failureXml.appendChild(messageXml)

for assembly_name in sorted(both_assemblies):
base_result = base_results_by_name[assembly_name]
diff_result = diff_results_by_name[assembly_name]
base_diff_are_equal = True

if base_result.returncode != diff_result.returncode:
base_diff_are_equal = False
elif base_result.returncode == 0 and diff_result.returncode == 0:
if base_result.output_file_hashsum != diff_result.output_file_hashsum:
base_diff_are_equal = False
if base_result.output_file_size_in_bytes != diff_result.output_file_size_in_bytes:
base_diff_are_equal = False
else:
base_diff_are_equal = False

base_result_string = json.dumps(base_result, cls=CrossGenResultEncoder, indent=2)
diff_result_string = json.dumps(diff_result, cls=CrossGenResultEncoder, indent=2)
message = 'Expected {0} got {1}'.format(base_result_string, diff_result_string)
testresult = root.createElement('test')
testresult.setAttribute('name', 'CrossgenCompile_{3}_Target_{0}_{1}_vs_{2}'.format(args.target_arch_os, base_result.compiler_arch_os, diff_result.compiler_arch_os, assembly_name))
testresult.setAttribute('type', 'Target_{0}'.format(args.target_arch_os))
testresult.setAttribute('method', '{0}_{1}'.format(base_result.compiler_arch_os, diff_result.compiler_arch_os))
testresult.setAttribute('time', '0')
if base_diff_are_equal:
testresult.setAttribute('result', 'Pass')
else:
testresult.setAttribute('result', 'Fail')

collection.appendChild(testresult)

if not base_diff_are_equal:
failureXml = root.createElement('failure')
failureXml.setAttribute('exception-type', 'MismatchOrReturnCodeFail')
testresult.appendChild(failureXml)

messageXml = root.createElement('message')
messageXml.appendChild(root.createTextNode(message))

failureXml.appendChild(messageXml)
messageXml = root.createElement('output')
messageXml.appendChild(root.createTextNode(message))

failureXml.appendChild(messageXml)

xml_str = root.toprettyxml(indent ="\t")

if output_file_type == FileTypes.NativeOrReadyToRunImage:
with open(args.testresultsxml, "w") as f:
f.write(xml_str)

if not did_compare:
sys.exit(1)

Expand Down

0 comments on commit 478571c

Please sign in to comment.