Skip to content

Commit

Permalink
junitrunner: Report suppressed and ignored tests as skipped.
Browse files Browse the repository at this point in the history 
    Fixes bazelbuild/bazel#6688
    (after java tools release)

    Closes #13717.

    PiperOrigin-RevId: 385953369
  • Loading branch information
Luca Di Grazia committed Sep 4, 2022
1 parent ada26d0 commit 9a4b652
Show file tree
Hide file tree
Showing 2 changed files with 139 additions and 35 deletions.
92 changes: 60 additions & 32 deletions ...java_tools/junitrunner/java/com/google/testing/junit/runner/model/AntXmlResultWriter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,16 +14,14 @@

package com.google.testing.junit.runner.model;

import com.google.common.base.Optional;
import com.google.common.base.Strings;
import com.google.common.collect.Iterables;

import org.joda.time.Interval;

import java.io.IOException;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.util.Map.Entry;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import javax.annotation.Nullable;
import javax.inject.Inject;

/**
* Writes the JUnit test nodes and their results into Ant-JUnit XML. Ant-JUnit XML is not a
Expand All @@ -34,6 +32,14 @@
public final class AntXmlResultWriter implements XmlResultWriter {
private static final String JUNIT_ELEMENT_TESTSUITES = "testsuites";
private static final String JUNIT_ELEMENT_TESTSUITE = "testsuite";
private static final String JUNIT_ELEMENT_TESTSUITE_PROPERTIES = "properties";
private static final String JUNIT_ELEMENT_TESTSUITE_SYSTEM_OUT = "system-out";
private static final String JUNIT_ELEMENT_TESTSUITE_SYSTEM_ERR = "system-err";
private static final String JUNIT_ELEMENT_PROPERTY = "property";
private static final String JUNIT_ELEMENT_TESTCASE = "testcase";
private static final String JUNIT_ELEMENT_FAILURE = "failure";
private static final String JUNIT_ELEMENT_SKIPPED = "skipped";

private static final String JUNIT_ATTR_TESTSUITE_ERRORS = "errors";
private static final String JUNIT_ATTR_TESTSUITE_FAILURES = "failures";
private static final String JUNIT_ATTR_TESTSUITE_HOSTNAME = "hostname";
Expand All @@ -43,14 +49,8 @@ public final class AntXmlResultWriter implements XmlResultWriter {
private static final String JUNIT_ATTR_TESTSUITE_TIMESTAMP = "timestamp";
private static final String JUNIT_ATTR_TESTSUITE_ID = "id";
private static final String JUNIT_ATTR_TESTSUITE_PACKAGE = "package";
private static final String JUNIT_ATTR_TESTSUITE_PROPERTIES = "properties";
private static final String JUNIT_ATTR_TESTSUITE_SYSTEM_OUT = "system-out";
private static final String JUNIT_ATTR_TESTSUITE_SYSTEM_ERR = "system-err";
private static final String JUNIT_ELEMENT_PROPERTY = "property";
private static final String JUNIT_ATTR_PROPERTY_NAME = "name";
private static final String JUNIT_ATTR_PROPERTY_VALUE = "value";
private static final String JUNIT_ELEMENT_TESTCASE = "testcase";
private static final String JUNIT_ELEMENT_FAILURE = "failure";
private static final String JUNIT_ATTR_FAILURE_MESSAGE = "message";
private static final String JUNIT_ATTR_FAILURE_TYPE = "type";
private static final String JUNIT_ATTR_TESTCASE_NAME = "name";
Expand All @@ -59,6 +59,10 @@ public final class AntXmlResultWriter implements XmlResultWriter {

private int testSuiteId;

@Inject
public AntXmlResultWriter() {}

@Override
public void writeTestSuites(XmlWriter writer, TestResult result) throws IOException {
testSuiteId = 0;
writer.startDocument();
Expand All @@ -73,7 +77,12 @@ public void writeTestSuites(XmlWriter writer, TestResult result) throws IOExcept
private void writeTestSuite(XmlWriter writer, TestResult result,
Iterable<Throwable> parentFailures)
throws IOException {
parentFailures = Iterables.concat(parentFailures, result.getFailures());
List<Throwable> allFailures = new ArrayList<>();
for (Throwable failure : parentFailures) {
allFailures.add(failure);
}
allFailures.addAll(result.getFailures());
parentFailures = allFailures;

writer.startElement(JUNIT_ELEMENT_TESTSUITE);

Expand All @@ -92,8 +101,8 @@ private void writeTestSuite(XmlWriter writer, TestResult result,
}

private void writeTestSuiteProperties(XmlWriter writer, TestResult result) throws IOException {
writer.startElement(JUNIT_ATTR_TESTSUITE_PROPERTIES);
for (Entry<String, String> entry : result.getProperties().entrySet()) {
writer.startElement(JUNIT_ELEMENT_TESTSUITE_PROPERTIES);
for (Map.Entry<String, String> entry : result.getProperties().entrySet()) {
writer.startElement(JUNIT_ELEMENT_PROPERTY);
writer.writeAttribute(JUNIT_ATTR_PROPERTY_NAME, entry.getKey());
writer.writeAttribute(JUNIT_ATTR_PROPERTY_VALUE, entry.getValue());
Expand All @@ -105,17 +114,20 @@ private void writeTestSuiteProperties(XmlWriter writer, TestResult result) throw
private void writeTestCases(XmlWriter writer, TestResult result,
Iterable<Throwable> parentFailures) throws IOException {
for (TestResult child : result.getChildResults()) {
if (child.getStatus().equals(TestResult.Status.FILTERED)) {
continue;
}
if (child.getChildResults().isEmpty()) {
writeTestCase(writer, child, parentFailures);
}
}
}

private void writeTestSuiteOutput(XmlWriter writer) throws IOException {
writer.startElement(JUNIT_ATTR_TESTSUITE_SYSTEM_OUT);
writer.startElement(JUNIT_ELEMENT_TESTSUITE_SYSTEM_OUT);
// TODO(bazel-team) - where to get this from?
writer.endElement();
writer.startElement(JUNIT_ATTR_TESTSUITE_SYSTEM_ERR);
writer.startElement(JUNIT_ELEMENT_TESTSUITE_SYSTEM_ERR);
// TODO(bazel-team) - where to get this from?
writer.endElement();
}
Expand All @@ -138,32 +150,48 @@ private void writeTestSuiteAttributes(XmlWriter writer, TestResult result) throw
writer.writeAttribute(JUNIT_ATTR_TESTSUITE_ID, this.testSuiteId++);
}

private static String getFormattedRunTime(Optional<Interval> runTimeInterval) {
return !runTimeInterval.isPresent() ? "0.0"
: String.valueOf(runTimeInterval.get().toDurationMillis() / 1000.0D);
private static String getFormattedRunTime(@Nullable TestInterval runTimeInterval) {
return runTimeInterval == null ? "0.0"
: String.valueOf(runTimeInterval.toDurationMillis() / 1000.0D);
}

private static String getFormattedTimestamp(Optional<Interval> runTimeInterval) {
return !runTimeInterval.isPresent() ? "" : runTimeInterval.get().getStart().toString();
private static String getFormattedTimestamp(@Nullable TestInterval runTimeInterval) {
return runTimeInterval == null ? "" : runTimeInterval.startInstantToString();
}

private void writeTestCase(XmlWriter writer, TestResult result,
Iterable<Throwable> parentFailures)
throws IOException {
writer.startElement(JUNIT_ELEMENT_TESTCASE);
writer.writeAttribute(JUNIT_ATTR_TESTCASE_NAME, result.getName());
writer.writeAttribute(JUNIT_ATTR_TESTCASE_CLASSNAME, result.getClassName());
writer.writeAttribute(JUNIT_ATTR_TESTCASE_TIME, getFormattedRunTime(
result.getRunTimeInterval()));

for (Throwable failure : parentFailures) {
writeThrowableToXmlWriter(writer, failure);
}

for (Throwable failure : Iterables.concat(parentFailures, result.getFailures())) {
writer.startElement(JUNIT_ELEMENT_FAILURE);
writer.writeAttribute(JUNIT_ATTR_FAILURE_MESSAGE, Strings.nullToEmpty(failure.getMessage()));
writer.writeAttribute(JUNIT_ATTR_FAILURE_TYPE, failure.getClass().getName());
writer.writeCharacters(formatStackTrace(failure));
for (Throwable failure : result.getFailures()) {
writeThrowableToXmlWriter(writer, failure);
}

if (result.getStatus().equals(TestResult.Status.SKIPPED)
|| result.getStatus().equals(TestResult.Status.SUPPRESSED)) {
writer.startElement(JUNIT_ELEMENT_SKIPPED);
writer.endElement();
}

writer.writeAttribute(JUNIT_ATTR_TESTCASE_NAME, result.getName());
writer.writeAttribute(JUNIT_ATTR_TESTCASE_CLASSNAME, result.getClassName());
writer.writeAttribute(JUNIT_ATTR_TESTCASE_TIME, getFormattedRunTime(
result.getRunTimeInterval()));
writer.endElement();
}

private static void writeThrowableToXmlWriter(XmlWriter writer, Throwable failure)
throws IOException {
writer.startElement(JUNIT_ELEMENT_FAILURE);
writer.writeAttribute(
JUNIT_ATTR_FAILURE_MESSAGE, (failure.getMessage() == null) ? "" : failure.getMessage());
writer.writeAttribute(JUNIT_ATTR_FAILURE_TYPE, failure.getClass().getName());
writer.writeCharacters(formatStackTrace(failure));
writer.endElement();
}

Expand Down
82 changes: 79 additions & 3 deletions ...s/junitrunner/javatests/com/google/testing/junit/runner/model/AntXmlResultWriterTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,18 +15,32 @@
package com.google.testing.junit.runner.model;

import static com.google.common.truth.Truth.assertThat;
import static com.google.testing.junit.runner.model.TestInstantUtil.testInstant;
import static java.nio.charset.StandardCharsets.UTF_8;

import com.google.testing.junit.runner.util.FakeTestClock;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.StringWriter;
import java.time.Duration;
import java.time.Instant;
import java.time.format.DateTimeFormatter;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.Description;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.NodeList;
import org.xml.sax.SAXException;

@RunWith(JUnit4.class)
public class AntXmlResultWriterTest {
private static final long NOW = 1;

private static TestSuiteNode root;
private static XmlWriter writer;
private static AntXmlResultWriter resultWriter;
Expand Down Expand Up @@ -69,9 +83,63 @@ public void testFilteredCasesNotWritten() throws IOException {
assertThat(resultXml).doesNotContain("<testcase name='testCase2'");
}

@Test
public void testWallTimeAndMonotonicTimestamp() throws Exception {
FakeTestClock clock = new FakeTestClock();
Instant startTime = Instant.ofEpochMilli(1560786184600L);
clock.setWallTimeOffset(startTime);
TestSuiteNode parent = createTestSuite();
TestCaseNode test = createTestCase(parent);

test.started(clock.now());
// wall time may appear to go back in time in exceptional cases (e.g. daylight saving time)
clock.advance(Duration.ofMillis(1L));
clock.setWallTimeOffset(startTime.minus(Duration.ofHours(1)));
test.finished(clock.now());

resultWriter.writeTestSuites(writer, root.getResult());

String resultXml = stringWriter.toString();
assertThat(resultXml).contains("time=");
assertThat(resultXml).contains("timestamp=");

Document document = parseXml(resultXml);
Element testSuites = document.getDocumentElement();
Element testSuite = (Element) testSuites.getElementsByTagName("testsuite").item(0);
assertThat(testSuite.getTagName()).isEqualTo("testsuite");
assertThat(testSuite.getAttribute("name"))
.isEqualTo("com.google.testing.junit.runner.model.TestCaseNodeTest$TestSuite");
assertThat(testSuite.getAttribute("time")).isEqualTo("0.001");
assertThat(
Instant.from(
DateTimeFormatter.ISO_DATE_TIME.parse(testSuite.getAttribute("timestamp"))))
.isEqualTo(startTime);
}

@Test
public void testSkippedOrSuppressedReportedAsSkipped() throws Exception {
TestSuiteNode parent = createTestSuite();
TestCaseNode skipped = createTestCase(parent);
skipped.started(testInstant(Instant.ofEpochMilli(1)));
skipped.testSkipped(testInstant(Instant.ofEpochMilli(2)));
TestCaseNode suppressed = createTestCase(parent);
suppressed.testSuppressed(testInstant(Instant.ofEpochMilli(4)));

resultWriter.writeTestSuites(writer, root.getResult());

Document document = parseXml(stringWriter.toString());
NodeList caseElems = document.getElementsByTagName("testcase");
assertThat(caseElems.getLength()).isEqualTo(2);
for (int i = 0; i < 2; i++) {
Element caseElem = (Element) caseElems.item(i);
NodeList skippedElems = caseElem.getElementsByTagName("skipped");
assertThat(skippedElems.getLength()).isEqualTo(1);
}
}

private void runToCompletion(TestCaseNode test) {
test.started(NOW);
test.finished(NOW + 1);
test.started(testInstant(Instant.ofEpochMilli(1)));
test.finished(testInstant(Instant.ofEpochMilli(2)));
}

private TestCaseNode createTestCase(TestSuiteNode parent) {
Expand All @@ -88,4 +156,12 @@ private TestSuiteNode createTestSuite() {
root.addTestSuite(parent);
return parent;
}

private static Document parseXml(String testXml)
throws SAXException, ParserConfigurationException, IOException {
DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
factory.setIgnoringElementContentWhitespace(true);
DocumentBuilder documentBuilder = factory.newDocumentBuilder();
return documentBuilder.parse(new ByteArrayInputStream(testXml.getBytes(UTF_8)));
}
}

0 comments on commit 9a4b652

Please sign in to comment.