diff --git a/tmt/result.py b/tmt/result.py index 77a5691d29..fb443b0b8f 100644 --- a/tmt/result.py +++ b/tmt/result.py @@ -82,6 +82,7 @@ class Result(tmt.utils.SerializableContainer): """ Describes what tmt knows about a single test result """ name: str + classname: str serialnumber: int = 0 fmf_id: Optional['tmt.base.FmfId'] = field( default=cast(Optional['tmt.base.FmfId'], None), @@ -115,6 +116,7 @@ def from_test( *, test: 'tmt.base.Test', result: ResultOutcome, + subresultname: Optional[str] = None, note: Optional[str] = None, ids: Optional[Dict[str, Optional[str]]] = None, log: Optional[List[Path]] = None, @@ -156,8 +158,12 @@ def from_test( guest_data = ResultGuestData(name=guest.name, role=guest.role) if guest is not None \ else ResultGuestData() + if not subresultname: + subresultname = test.name + _result = Result( - name=test.name, + classname=test.name, + name=subresultname, serialnumber=test.serialnumber, fmf_id=test.fmf_id, result=result, diff --git a/tmt/steps/execute/__init__.py b/tmt/steps/execute/__init__.py index 79ed94ae59..b0cde06b88 100644 --- a/tmt/steps/execute/__init__.py +++ b/tmt/steps/execute/__init__.py @@ -1,6 +1,7 @@ import copy import dataclasses import datetime +import os from dataclasses import dataclass from typing import TYPE_CHECKING, Any, List, Optional, Tuple, Type, cast @@ -265,8 +266,7 @@ def load_tmt_report_results(self, test: "tmt.Test", guest: Guest) -> List["tmt.R or an empty list if the file does not exist. """ report_result_path = self.data_path(test, guest, full=True) \ - / tmt.steps.execute.TEST_DATA \ - / TMT_REPORT_RESULT_SCRIPT.created_file + / tmt.steps.execute.TEST_DATA # Nothing to do if there's no result file if not report_result_path.exists(): @@ -274,33 +274,73 @@ def load_tmt_report_results(self, test: "tmt.Test", guest: Guest) -> List["tmt.R return [] # Check the test result - self.debug(f"tmt-report-results file '{report_result_path} detected.") + self.debug(f"tmt-report-results file '{report_result_path}' detected.") - with open(report_result_path) as result_file: - result_list = [line for line in result_file.readlines() if "TESTRESULT" in line] - if not result_list: - raise tmt.utils.ExecuteError( - f"Test result not found in result file '{report_result_path}'.") - result = result_list[0].split("=")[1].strip() + if not report_result_path.glob(r'*restraint-result.*'): + self.debug(f"tmt-report-results file '{report_result_path}' is empty.") + return [] - # Map the restraint result to the corresponding tmt value - actual_result = ResultOutcome.ERROR - note: Optional[str] = None + all_results = [] + for f in sorted(report_result_path.glob(r'*restraint-result.*'), \ + key=os.path.getmtime): + self.debug(f"tmt-report-results file '{f}' detected.") + with open(f) as result_file: + result_list = [line for line in result_file.readlines() if "TESTRESULT" in line] + with open(f) as result_file: + log_list = [line for line in result_file.readlines() if "OUTPUTFILE" in line] + with open(f) as result_file: + name_list = [line for line in result_file.readlines() if "TESTNAME" in line] + + if not result_list: + raise tmt.utils.ExecuteError( + f"Test result not found in result file '{f}'.") + else: + if len(result_list[0].split("=")) == 2: + result = result_list[0].split("=")[1].strip() - try: - actual_result = ResultOutcome(result.lower()) - except ValueError: - if result == 'SKIP': - actual_result = ResultOutcome.INFO + if not log_list: + self.debug(f"Test log not found in result file '{f}'.") + logfile=[self.data_path(test, guest, TEST_OUTPUT_FILENAME)] else: - note = f"invalid test result '{result}' in result file" - - return [tmt.Result.from_test( - test=test, - result=actual_result, - log=[self.data_path(test, guest, TEST_OUTPUT_FILENAME)], - note=note, - guest=guest)] + if len(log_list[0].split("=")) == 2: + logname = log_list[0].split("=")[1].strip() + if logname == '': + logfile = \ + [self.data_path(test, guest, TEST_OUTPUT_FILENAME)] + else: + logfile = [self.data_path(test, guest, full=True) \ + / tmt.steps.execute.TEST_DATA / logname] + + if not name_list: + self.debug(f"Test name not found in result file '{f}'.") + resultname = test.name + else: + if len(name_list[0].split("=")) == 2: + resultname = name_list[0].split("=")[1].strip() + else: + resultname = test.name + + # Map the restraint result to the corresponding tmt value + actual_result = ResultOutcome.ERROR + note: Optional[str] = None + + try: + actual_result = ResultOutcome(result.lower()) + except ValueError: + if result == 'SKIP': + actual_result = ResultOutcome.INFO + else: + note = f"invalid test result '{result}' in result file" + + all_results.append(tmt.Result.from_test( + test=test, + subresultname=resultname, + result=actual_result, + log=logfile, + note=note, + guest=guest)) + + return all_results def load_custom_results(self, test: "tmt.Test", guest: Guest) -> List["tmt.Result"]: """ diff --git a/tmt/steps/execute/scripts/tmt-report-result b/tmt/steps/execute/scripts/tmt-report-result index 698983ca57..255b7a4a1f 100755 --- a/tmt/steps/execute/scripts/tmt-report-result +++ b/tmt/steps/execute/scripts/tmt-report-result @@ -1,13 +1,6 @@ #!/bin/bash set -o errexit -o pipefail -o noclobber -o nounset -server= -outputFile= -disablePlugin= -port= -message= -help=False - die() { echo "$*" >&2; exit 2; } # complain to STDERR and exit with error needs_arg() { if [ -z "$OPTARG" ]; then die "No arg for --$OPT option"; fi; } @@ -38,12 +31,41 @@ check_opt_args () { shift $((OPTIND-1)) # remove parsed options and args from $@ list position=$((OPTIND-1)) } -# Options wrapped in quotes to ensure -t/--message argument is parsed as a string phrase -# rather than just the first string until a whitespace is encountered. + +write_report_file () { + #rm -f $REPORT_RESULT_OUTPUTFILE + if [ -e $REPORT_RESULT_CNT_FILE ] + then + local cnt=$(cat $REPORT_RESULT_CNT_FILE | wc -l) + else + local cnt=0 + fi + echo "SERVER=$server" >> $REPORT_RESULT_OUTPUTFILE.$cnt + echo "PORT=$port" >> $REPORT_RESULT_OUTPUTFILE.$cnt + echo "MESSAGE=$message" >> $REPORT_RESULT_OUTPUTFILE.$cnt + echo "OUTPUTFILE=$resultfile" >> $REPORT_RESULT_OUTPUTFILE.$cnt + echo "DISABLEPLUGIN=$disablePlugin" >> $REPORT_RESULT_OUTPUTFILE.$cnt + echo "TESTNAME=$TESTNAME" >> $REPORT_RESULT_OUTPUTFILE.$cnt + echo "TESTNAME=$TESTNAME" >> $REPORT_RESULT_CNT_FILE + echo "TESTRESULT=$TESTRESULT" >> $REPORT_RESULT_OUTPUTFILE.$cnt + echo "METRIC=$METRIC" >> $REPORT_RESULT_OUTPUTFILE.$cnt +} + +server= +outputFile= +disablePlugin= +port= +message= +help=False + +# Options wrapped in quotes to ensure -t/--message argument is parsed as a +# string phrase rather than just the first string until a whitespace +# is encountered. check_opt_args "$@" shift $position # remove parsed options and args from $@ list # Return help options when command issued with no options or arguments. -if [ $# -lt 2 ] || [ $help == True ]; then +if ([ $# -lt 2 ] || [ $help == True ]) && [[ $0 =~ "rstrnt-report-result" ]] +then echo "Usage:" echo " rstrnt-report-result [OPTION?] TASK_PATH RESULT [SCORE]" echo "" @@ -62,12 +84,27 @@ if [ $# -lt 2 ] || [ $help == True ]; then echo " -o, --outputfile=FILE Log to upload with result, \$OUTPUTFILE is used by default" echo " -p, --disable-plugin=PLUGIN don't run plugin on server side" echo " --no-plugins don't run any plugins on server side" - echo "" + exit 1 +fi +if ([ $# -lt 2 ] || [ $help == True ]) && [[ $0 =~ "rhts-report-result" ]] +then + echo "Usage: rhts-report-result TESTNAME TESTRESULT LOGFILE [METRIC]" + echo "where TESTNAME is the name of the test result being reported" + echo " TESTRESULT should be PASS or FAIL" + echo " LOGFILE is the name of the log file to be uploaded" + echo " METRIC is an optional numeric metric (e.g. a performance figure)" exit 1 fi TESTNAME=$1 TESTRESULT=$2 shift 2 #$((OPTIND-1)) # remove parsed options and args from $@ list + +# the third postion parameter of the rhts helper is log file +if [[ $0 =~ "rhts-report-result" ]] +then + outputFile=$1 + shift 1 +fi METRIC= if [ $# -gt 0 ];then value1=$1 @@ -79,48 +116,16 @@ if [ $# -gt 0 ];then check_opt_args "$@" fi -write_report_file () { - rm -f $REPORT_RESULT_OUTPUTFILE - echo "SERVER=$server" >> $REPORT_RESULT_OUTPUTFILE - echo "PORT=$port" >> $REPORT_RESULT_OUTPUTFILE - echo "MESSAGE=$message" >> $REPORT_RESULT_OUTPUTFILE - echo "OUTPUTFILE=$outputFile" >> $REPORT_RESULT_OUTPUTFILE - echo "DISABLEPLUGIN=$disablePlugin" >> $REPORT_RESULT_OUTPUTFILE - echo "TESTNAME=$TESTNAME" >> $REPORT_RESULT_OUTPUTFILE - echo "TESTRESULT=$TESTRESULT" >> $REPORT_RESULT_OUTPUTFILE - echo "METRIC=$METRIC" >> $REPORT_RESULT_OUTPUTFILE -} - +if [ -n "$outputFile" ] && [ -f $outputFile ] +then + resultfile=$(mktemp -u result.XXXXXX) + cp -f "$outputFile" "$TMT_TEST_DATA"/$resultfile +else + resultfile="" +fi -declare -A RESULTS_HIERARCHY=( ["FAIL"]=4 ["WARN"]=3 ["PASS"]=2 ["SKIP"]=1 ) # Create the output file in which to store the results. REPORT_RESULT_OUTPUTFILE="$TMT_TEST_DATA/restraint-result" -if [ -e "$REPORT_RESULT_OUTPUTFILE" ]; then - # Check the existing output file. - # If new result is higher on the hierarchy - # or no result is contained in the file - # then overwrite the file with the new - # report contents. - fileResult=$(/usr/bin/grep "TESTRESULT" $REPORT_RESULT_OUTPUTFILE | - cut -d = -f 2) - if [ ! $fileResult ]; then - write_report_file; - exit 0 - elif [ ! $TESTRESULT ]; then - exit 0 - fi - fileHierarchy="${RESULTS_HIERARCHY[$fileResult]}" - # Compare the current result hierarchy against - # the hierarchy result retrieved from the file. - # Overwrite the file if the current hierachy is higher. - thisResultHierarchy="${RESULTS_HIERARCHY[$TESTRESULT]}" - if [ $thisResultHierarchy -gt $fileHierarchy ]; then - write_report_file - exit 0 - fi - -else - # Create the output file with the report contents. - write_report_file - exit 0 -fi +REPORT_RESULT_CNT_FILE="$TMT_TEST_DATA/restraint-result-cnt" +write_report_file +exit 0 diff --git a/tmt/steps/report/junit.py b/tmt/steps/report/junit.py index 2f32c49a2f..636b3a18aa 100644 --- a/tmt/steps/report/junit.py +++ b/tmt/steps/report/junit.py @@ -68,7 +68,7 @@ def make_junit_xml(report: "tmt.steps.report.ReportPlugin") -> JunitTestSuite: main_log = None case = junit_xml.TestCase( result.name, - classname=None, + classname=result.classname, elapsed_sec=duration_to_seconds(result.duration), stdout=main_log) # Map tmt OUTCOME to JUnit states