From ee24df53993915647e4c586a5327308b7b8222a7 Mon Sep 17 00:00:00 2001
From: Enrico Minack <github@enrico.minack.dev>
Date: Mon, 13 Jan 2025 11:24:54 +0100
Subject: [PATCH] Add more is_* properties to JUnit and XUnit2 TestCase
- Add is_failure and is_error to JUnit TestCase
- Add is_rerun and is_flaky to XUnit2 TestCase
- Provide access to XUnit2 interim (rerun & flaky) results
---
junitparser/junitparser.py | 31 +++++++------
junitparser/xunit2.py | 56 +++++++++++++++--------
tests/test_xunit2.py | 92 +++++++++++++++++++++++++++++++++++---
3 files changed, 142 insertions(+), 37 deletions(-)
diff --git a/junitparser/junitparser.py b/junitparser/junitparser.py
index be6109a..431833a 100644
--- a/junitparser/junitparser.py
+++ b/junitparser/junitparser.py
@@ -321,6 +321,9 @@ class TestCase(Element):
time = FloatAttr()
__test__ = False
+ # JUnit TestCase children are final results, SystemOut and SystemErr
+ ITER_TYPES = {t._tag: t for t in set.union(FINAL_RESULTS, {SystemOut, SystemErr})}
+
def __init__(self, name: str = None, classname: str = None, time: float = None):
super().__init__(self._tag)
if name is not None:
@@ -334,11 +337,9 @@ def __hash__(self):
return super().__hash__()
def __iter__(self):
- all_types = set.union(FINAL_RESULTS, {SystemOut}, {SystemErr})
for elem in self._elem.iter():
- for entry_type in all_types:
- if elem.tag == entry_type._tag:
- yield entry_type.fromelem(elem)
+ if elem.tag in self.ITER_TYPES:
+ yield self.ITER_TYPES[elem.tag].fromelem(elem)
def __eq__(self, other):
# TODO: May not work correctly if unreliable hash method is used.
@@ -349,23 +350,25 @@ def is_passed(self):
"""Whether this testcase was a success (i.e. if it isn't skipped, failed, or errored)."""
return not self.result
+ @property
+ def is_failure(self):
+ """Whether this testcase failed."""
+ return any(isinstance(r, Failure) for r in self.result)
+
+ @property
+ def is_error(self):
+ """Whether this testcase errored."""
+ return any(isinstance(r, Error) for r in self.result)
+
@property
def is_skipped(self):
"""Whether this testcase was skipped."""
- for r in self.result:
- if isinstance(r, Skipped):
- return True
- return False
+ return any(isinstance(r, Skipped) for r in self.result)
@property
def result(self) -> List[FinalResult]:
"""A list of :class:`Failure`, :class:`Skipped`, or :class:`Error` objects."""
- results = []
- for entry in self:
- if isinstance(entry, FinalResult):
- results.append(entry)
-
- return results
+ return [entry for entry in self if isinstance(entry, FinalResult)]
@result.setter
def result(self, value: Union[FinalResult, List[FinalResult]]):
diff --git a/junitparser/xunit2.py b/junitparser/xunit2.py
index 236ede4..500793f 100644
--- a/junitparser/xunit2.py
+++ b/junitparser/xunit2.py
@@ -13,12 +13,9 @@
"""
import itertools
-from typing import List, TypeVar
+from typing import List, Type, TypeVar
from . import junitparser
-T = TypeVar("T")
-
-
class TestSuite(junitparser.TestSuite):
"""TestSuite for Pytest, with some different attributes."""
@@ -175,31 +172,54 @@ class FlakyError(InterimResult):
_tag = "flakyError"
+RERUN_RESULTS = {RerunFailure, RerunError, FlakyFailure, FlakyError}
+
+
+R = TypeVar("R", bound=InterimResult)
+
+
class TestCase(junitparser.TestCase):
group = junitparser.Attr()
- def _rerun_results(self, _type: T) -> List[T]:
- elems = self.iterchildren(_type)
- results = []
- for elem in elems:
- results.append(_type.fromelem(elem))
- return results
+ # XUnit2 TestCase children are JUnit children and rerun results
+ ITER_TYPES = {t._tag: t for t in list(junitparser.TestCase.ITER_TYPES.values()) + list(RERUN_RESULTS)}
- def rerun_failures(self):
+ def _interim_results(self, _type: Type[R]) -> List[R]:
+ return [entry for entry in self if isinstance(entry, _type)]
+
+ @property
+ def interim_result(self) -> List[InterimResult]:
+ """
+ A list of interim results: :class:`RerunFailure`, :class:`RerunError`, :class:`FlakyFailure`, or :class:`FlakyError` objects.
+ This is complementary to the result property returning final results.
+ """
+ return self._interim_results(InterimResult)
+
+ def rerun_failures(self) -> List[RerunFailure]:
"""<rerunFailure>"""
- return self._rerun_results(RerunFailure)
+ return self._interim_results(RerunFailure)
- def rerun_errors(self):
+ def rerun_errors(self) -> List[RerunError]:
"""<rerunError>"""
- return self._rerun_results(RerunError)
+ return self._interim_results(RerunError)
- def flaky_failures(self):
+ def flaky_failures(self) -> List[FlakyFailure]:
"""<flakyFailure>"""
- return self._rerun_results(FlakyFailure)
+ return self._interim_results(FlakyFailure)
- def flaky_errors(self):
+ def flaky_errors(self) -> List[FlakyError]:
"""<flakyError>"""
- return self._rerun_results(FlakyError)
+ return self._interim_results(FlakyError)
+
+ @property
+ def is_rerun(self) -> bool:
+ """Whether this testcase is rerun, i.e., there are rerun failures or errors."""
+ return any(self.rerun_failures()) or any(self.rerun_errors())
+
+ @property
+ def is_flaky(self) -> bool:
+ """Whether this testcase is flaky, i.e., there are flaky failures or errors."""
+ return any(self.flaky_failures()) or any(self.flaky_errors())
def add_interim_result(self, result: InterimResult):
"""Append an interim (rerun or flaky) result to the testcase. A testcase can have multiple interim results."""
diff --git a/tests/test_xunit2.py b/tests/test_xunit2.py
index 07ccf71..da50140 100644
--- a/tests/test_xunit2.py
+++ b/tests/test_xunit2.py
@@ -1,12 +1,12 @@
# -*- coding: utf-8 -*-
-from junitparser.xunit2 import JUnitXml, TestSuite, TestCase, RerunFailure
+from junitparser.xunit2 import JUnitXml, TestSuite, TestCase, RerunFailure, RerunError, FlakyFailure, FlakyError
from junitparser import Failure
from copy import deepcopy
class Test_TestCase:
- def test_case_fromstring(self):
+ def test_case_rerun_fromstring(self):
text = """<testcase name="testname">
<failure message="failure message" type="FailureType"/>
<rerunFailure message="Not found" type="404">
@@ -16,28 +16,110 @@ def test_case_fromstring(self):
<system-err>Error del servidor</system-err>
<stackTrace>Stacktrace</stackTrace>
</rerunFailure>
+ <rerunError message="Setup error"/>
<system-out>System out</system-out>
<system-err>System err</system-err>
</testcase>"""
case = TestCase.fromstring(text)
assert case.name == "testname"
+ assert len(case.result) == 1
assert isinstance(case.result[0], Failure)
assert case.system_out == "System out"
assert case.system_err == "System err"
+ assert case.is_passed == False
+ assert case.is_failure == True
+ assert case.is_error == False
+ assert case.is_skipped == False
+ assert case.is_rerun == True
+ assert case.is_flaky == False
+
+ interim_results = case.interim_result
+ assert len(interim_results) == 3
+ assert isinstance(interim_results[0], RerunFailure)
+ assert isinstance(interim_results[1], RerunFailure)
+ assert isinstance(interim_results[2], RerunError)
+
rerun_failures = case.rerun_failures()
assert len(rerun_failures) == 2
+ assert isinstance(rerun_failures[0], RerunFailure)
assert rerun_failures[0].message == "Not found"
assert rerun_failures[0].stack_trace is None
assert rerun_failures[0].system_out == "No ha encontrado"
assert rerun_failures[0].system_err is None
+ assert isinstance(rerun_failures[1], RerunFailure)
assert rerun_failures[1].message == "Server error"
assert rerun_failures[1].stack_trace == "Stacktrace"
assert rerun_failures[1].system_out is None
assert rerun_failures[1].system_err == "Error del servidor"
- assert len(case.rerun_errors()) == 0
+
+ rerun_errors = case.rerun_errors()
+ assert len(rerun_errors) == 1
+ assert isinstance(rerun_errors[0], RerunError)
+ assert rerun_errors[0].message == "Setup error"
+ assert rerun_errors[0].stack_trace is None
+ assert rerun_errors[0].system_out is None
+ assert rerun_errors[0].system_err is None
+
assert len(case.flaky_failures()) == 0
+
assert len(case.flaky_errors()) == 0
+ def test_case_flaky_fromstring(self):
+ text = """<testcase name="testname">
+ <flakyFailure message="Not found" type="404">
+ <system-out>No ha encontrado</system-out>
+ </flakyFailure>
+ <flakyFailure message="Server error" type="500">
+ <system-err>Error del servidor</system-err>
+ <stackTrace>Stacktrace</stackTrace>
+ </flakyFailure>
+ <flakyError message="Setup error"/>
+ <system-out>System out</system-out>
+ <system-err>System err</system-err>
+ </testcase>"""
+ case = TestCase.fromstring(text)
+ assert case.name == "testname"
+ assert len(case.result) == 0
+ assert case.system_out == "System out"
+ assert case.system_err == "System err"
+ assert case.is_passed == True
+ assert case.is_failure == False
+ assert case.is_error == False
+ assert case.is_skipped == False
+ assert case.is_rerun == False
+ assert case.is_flaky == True
+
+ interim_results = case.interim_result
+ assert len(interim_results) == 3
+ assert isinstance(interim_results[0], FlakyFailure)
+ assert isinstance(interim_results[1], FlakyFailure)
+ assert isinstance(interim_results[2], FlakyError)
+
+ assert len(case.rerun_failures()) == 0
+
+ assert len(case.rerun_errors()) == 0
+
+ flaky_failures = case.flaky_failures()
+ assert len(flaky_failures) == 2
+ assert isinstance(flaky_failures[0], FlakyFailure)
+ assert flaky_failures[0].message == "Not found"
+ assert flaky_failures[0].stack_trace is None
+ assert flaky_failures[0].system_out == "No ha encontrado"
+ assert flaky_failures[0].system_err is None
+ assert isinstance(flaky_failures[1], FlakyFailure)
+ assert flaky_failures[1].message == "Server error"
+ assert flaky_failures[1].stack_trace == "Stacktrace"
+ assert flaky_failures[1].system_out is None
+ assert flaky_failures[1].system_err == "Error del servidor"
+
+ flaky_errors = case.flaky_errors()
+ assert len(flaky_errors) == 1
+ assert isinstance(flaky_errors[0], FlakyError)
+ assert flaky_errors[0].message == "Setup error"
+ assert flaky_errors[0].stack_trace is None
+ assert flaky_errors[0].system_out is None
+ assert flaky_errors[0].system_err is None
+
def test_case_rerun(self):
case = TestCase("testname")
rerun_failure = RerunFailure("Not found", "404")
@@ -46,14 +128,14 @@ def test_case_rerun(self):
rerun_failure.stack_trace = "Stack"
rerun_failure.system_err = "E404"
rerun_failure.system_out = "NOT FOUND"
- case.add_rerun_result(rerun_failure)
+ case.add_interim_result(rerun_failure)
assert len(case.rerun_failures()) == 1
# Interesting, same object is only added once by xml libs
failure2 = deepcopy(rerun_failure)
failure2.stack_trace = "Stack2"
failure2.system_err = "E401"
failure2.system_out = "401 Error"
- case.add_rerun_result(failure2)
+ case.add_interim_result(failure2)
assert len(case.rerun_failures()) == 2