Add an XCTest observer to swift_test targets that generates a JUnit…

…-style XML log at the path in the `XML_OUTPUT_PATH` environment variable defined by Bazel.

Due to differences between the open-source XCTest and Xcode's XCTest, only Darwin-based platforms can distinguish "skipped" tests from "passing" tests at this time (and even on that platform, it can only do so by referencing private APIs).

PiperOrigin-RevId: 437304646
(cherry picked from commit 497f079)
Signed-off-by: Brentley Jones <github@brentleyjones.com>

swift/internal/derived_files.bzl

@@ -438,6 +438,18 @@ def _intermediate_swift_const_values_file(actions, target_name, src):
         paths.join(dirname, "{}.swiftconstvalues".format(basename)),
     )
 
+def _xctest_bundle(actions, target_name):
+    """Declares a directory for the `.xctest` bundle of a Darwin `swift_test`.
+
+    Args:
+        actions: The context's actions object.
+        target_name: The name of the target being built.
+
+    Returns:
+        The declared `File`.
+    """
+    return actions.declare_directory("{}.xctest".format(target_name))
+
 def _xctest_runner_script(actions, target_name):
     """Declares a file for the script that runs an `.xctest` bundle on Darwin.
 
@@ -474,6 +486,7 @@ derived_files = struct(
     vfsoverlay = _vfsoverlay,
     whole_module_object_file = _whole_module_object_file,
     swift_const_values_file = _swift_const_values_file,
+    xctest_bundle = _xctest_bundle,
     xctest_runner_script = _xctest_runner_script,
     generated_header = _declare_validated_generated_header,
 )

swift/internal/swift_test.bzl

@@ -64,15 +64,55 @@ def _maybe_parse_as_library_copts(srcs):
                            srcs[0].basename != "main.swift"
     return ["-parse-as-library"] if use_parse_as_library else []
 
-def _create_xctest_runner(name, actions, executable, xctest_runner_template):
+def _create_xctest_bundle(name, actions, binary):
+    """Creates an `.xctest` bundle that contains the given binary.
+
+    Args:
+        name: The name of the target being built, which will be used as the
+            basename of the bundle (followed by the .xctest bundle extension).
+        actions: The context's actions object.
+        binary: The binary that will be copied into the test bundle.
+
+    Returns:
+        A `File` (tree artifact) representing the `.xctest` bundle.
+    """
+    xctest_bundle = derived_files.xctest_bundle(
+        actions = actions,
+        target_name = name,
+    )
+
+    args = actions.args()
+    args.add(xctest_bundle.path)
+    args.add(binary)
+
+    # When XCTest loads this bundle, it will create an instance of this class
+    # which will register the observer that writes the XML output.
+    plist = '{ NSPrincipalClass = "BazelXMLTestObserverRegistration"; }'
+
+    actions.run_shell(
+        arguments = [args],
+        command = (
+            'mkdir -p "$1/Contents/MacOS" && ' +
+            'cp "$2" "$1/Contents/MacOS" && ' +
+            'echo \'{}\' > "$1/Contents/Info.plist"'.format(plist)
+        ),
+        inputs = [binary],
+        mnemonic = "SwiftCreateTestBundle",
+        outputs = [xctest_bundle],
+        progress_message = "Creating test bundle for {}".format(name),
+    )
+
+    return xctest_bundle
+
+def _create_xctest_runner(name, actions, bundle, xctest_runner_template):
     """Creates a script that will launch `xctest` with the given test bundle.
 
     Args:
         name: The name of the target being built, which will be used as the
             basename of the test runner script.
         actions: The context's actions object.
-        executable: The `File` representing the executable inside the `.xctest`
-            bundle that should be executed.
+        bundle: The `File` representing the `.xctest` bundle that should be
+            executed.
         xctest_runner_template: The `File` that will be used as a template to
             generate the test runner shell script.
 
@@ -90,7 +130,7 @@ def _create_xctest_runner(name, actions, executable, xctest_runner_template):
         output = xctest_runner,
         template = xctest_runner_template,
         substitutions = {
-            "%executable%": executable.short_path,
+            "%bundle%": bundle.short_path,
         },
     )
 
@@ -297,7 +337,7 @@ def _swift_test_impl(ctx):
         # This is already collected from `linking_context`.
         compilation_outputs = None,
         deps = ctx.attr.deps + extra_link_deps,
-        name = "{0}.xctest/Contents/MacOS/{0}".format(ctx.label.name) if is_bundled else ctx.label.name,
+        name = ctx.label.name,
         output_type = "executable",
         owner = ctx.label,
         stamp = ctx.attr.stamp,
@@ -313,13 +353,18 @@ def _swift_test_impl(ctx):
     # script that launches it via `xctest`. Otherwise, just use the binary
     # itself as the executable to launch.
     if is_bundled:
+        xctest_bundle = _create_xctest_bundle(
+            name = ctx.label.name,
+            actions = ctx.actions,
+            binary = linking_outputs.executable,
+        )
         xctest_runner = _create_xctest_runner(
             name = ctx.label.name,
             actions = ctx.actions,
-            executable = linking_outputs.executable,
+            bundle = xctest_bundle,
             xctest_runner_template = ctx.file._xctest_runner_template,
         )
-        additional_test_outputs = [linking_outputs.executable]
+        additional_test_outputs = [xctest_bundle]
         executable = xctest_runner
     else:
         additional_test_outputs = []

tools/test_discoverer/TestPrinter.swift

@@ -134,6 +134,7 @@ struct TestPrinter {
   /// Prints the main test runner to a Swift source file.
   func printTestRunner(toFileAt url: URL) {
     var contents = """
+      import BazelTestObservation
       import XCTest
 
       @main
@@ -153,6 +154,9 @@ struct TestPrinter {
     }
 
     contents += """
+          if let xmlObserver = BazelXMLTestObserver.default {
+            XCTestObservationCenter.shared.addTestObserver(xmlObserver)
+          }
           XCTMain(tests)
         }
       }

tools/test_observer/BUILD

@@ -0,0 +1,12 @@
+load("//swift:swift.bzl", "swift_library")
+
+swift_library(
+    name = "test_observer",
+    srcs = [
+        "BazelXMLTestObserver.swift",
+        "BazelXMLTestObserverRegistration.swift",
+        "StringInterpolation+XMLEscaping.swift",
+    ],
+    module_name = "BazelTestObservation",
+    visibility = ["//visibility:public"],
+)

tools/test_observer/BazelXMLTestObserver.swift

@@ -0,0 +1,210 @@
+// Copyright 2022 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import Foundation
+import XCTest
+
+/// An XCTest observer that generates an XML file in the format described by the
+/// [JUnit test result schema](https://windyroad.com.au/dl/Open%20Source/JUnit.xsd).
+public final class BazelXMLTestObserver: NSObject {
+  /// The file handle to which the XML content will be written.
+  private let fileHandle: FileHandle
+
+  /// The current indentation to print before each line, as UTF-8 code units.
+  private var indentation: Data
+
+  /// The default XML-generating XCTest observer, which determines the output file based on the
+  /// value of the `XML_OUTPUT_FILE` environment variable.
+  ///
+  /// If the `XML_OUTPUT_FILE` environment variable is not set or the file at that path could not be
+  /// created and opened for writing, the value of this property will be nil.
+  public static let `default`: BazelXMLTestObserver? = {
+    guard
+      let outputPath = ProcessInfo.processInfo.environment["XML_OUTPUT_FILE"],
+      FileManager.default.createFile(atPath: outputPath, contents: nil, attributes: nil),
+      let fileHandle = FileHandle(forWritingAtPath: outputPath)
+    else {
+      return nil
+    }
+    return .init(fileHandle: fileHandle)
+  }()
+
+  /// Creates a new XML-generating XCTest observer that writes its content to the given file handle.
+  private init(fileHandle: FileHandle) {
+    self.fileHandle = fileHandle
+    self.indentation = Data()
+  }
+
+  /// Writes the given string to the observer's file handle.
+  private func writeLine<S: StringProtocol>(_ string: S) {
+    if !indentation.isEmpty {
+      fileHandle.write(indentation)
+    }
+    fileHandle.write(string.data(using: .utf8)!)  // Conversion to UTF-8 cannot fail.
+    fileHandle.write(Data([UInt8(ascii: "\n")]))
+  }
+
+  /// Increases the current indentation level by two spaces.
+  private func indent() {
+    indentation.append(contentsOf: [UInt8(ascii: " "), UInt8(ascii: " ")])
+  }
+
+  /// Reduces the current indentation level by two spaces.
+  private func dedent() {
+    indentation.removeLast(2)
+  }
+
+  /// Canonicalizes the name of the test case for printing into the XML file.
+  ///
+  /// The canonical name of the test is `TestClass.testMethod` (i.e., Swift-style syntax). When
+  /// running tests under the Objective-C runtime, the test cases will have Objective-C-style names
+  /// (i.e., `-[TestClass testMethod]`), so this method converts those to the desired syntax.
+  ///
+  /// Any test name that does not match one of those two syntaxes is returned unchanged.
+  private func canonicalizedName(of testCase: XCTestCase) -> String {
+    let name = testCase.name
+    guard name.hasPrefix("-[") && name.hasSuffix("]") else {
+      return name
+    }
+
+    let trimmedName = name.dropFirst(2).dropLast()
+    guard let spaceIndex = trimmedName.lastIndex(of: " ") else {
+      return String(trimmedName)
+    }
+
+    return "\(trimmedName[..<spaceIndex]).\(trimmedName[trimmedName.index(after: spaceIndex)...])"
+  }
+}
+
+extension BazelXMLTestObserver: XCTestObservation {
+  public func testBundleWillStart(_ testBundle: Bundle) {
+    writeLine(#"<?xml version="1.0" encoding="utf-8"?>"#)
+    writeLine("<testsuites>")
+    indent()
+  }
+
+  public func testBundleDidFinish(_ testBundle: Bundle) {
+    dedent()
+    writeLine("</testsuites>")
+  }
+
+  public func testSuiteWillStart(_ testSuite: XCTestSuite) {
+    writeLine(
+      #"<testsuite name="\#(xmlEscaping: testSuite.name)" tests="\#(testSuite.testCaseCount)">"#)
+    indent()
+  }
+
+  public func testSuiteDidFinish(_ testSuite: XCTestSuite) {
+    dedent()
+    writeLine("</testsuite>")
+  }
+
+  public func testCaseWillStart(_ testCase: XCTestCase) {
+    writeLine(
+      #"<testcase name="\#(xmlEscaping: canonicalizedName(of: testCase))" status="run" "#
+      + #"result="completed">"#)
+    indent()
+  }
+
+  public func testCaseDidFinish(_ testCase: XCTestCase) {
+    dedent()
+    writeLine("</testcase>")
+  }
+
+  // On platforms with the Objective-C runtime, we use the richer `XCTIssue`-based APIs. Anywhere
+  // else, we're building with the open-source version of XCTest which has only the older
+  // `didFailWithDescription` API.
+  #if canImport(ObjectiveC)
+    public func testCase(_ testCase: XCTestCase, didRecord issue: XCTIssue) {
+      let tag: String
+      switch issue.type {
+      case .assertionFailure, .performanceRegression, .unmatchedExpectedFailure:
+        tag = "failure"
+      case .system, .thrownError, .uncaughtException:
+        tag = "error"
+      @unknown default:
+        tag = "failure"
+      }
+
+      writeLine(#"<\#(tag) message="\#(xmlEscaping: issue.compactDescription)"/>"#)
+    }
+  #else
+    public func testCase(
+      _ testCase: XCTestCase,
+      didFailWithDescription description: String,
+      inFile filePath: String?,
+      atLine lineNumber: Int
+    ) {
+      let tag = description.hasPrefix(#"threw error ""#) ? "error" : "failure"
+      writeLine(#"<\#(tag) message="\#(xmlEscaping: description)"/>"#)
+    }
+  #endif
+}
+
+// Hacks ahead! XCTest does not declare the methods that it uses to notify observers of skipped
+// tests as part of the public `XCTestObservation` protocol. Instead, they are only available on
+// various framework-internal protocols that XCTest checks for conformance against at runtime.
+//
+// On Darwin platforms, thanks to the Objective-C runtime, we can declare protocols with the same
+// names in our library and implement those methods, and XCTest will call them so that we can log
+// the skipped tests in our output. Note that we have to re-specify the protocol name in the `@objc`
+// attribute to remove the module name for the runtime.
+//
+// On non-Darwin platforms, we don't have an escape hatch because XCTest is implemented in pure
+// Swift and we can't play the same runtime games, so skipped tests simply get tracked as "passing"
+// there.
+#if canImport(ObjectiveC)
+  /// Declares the observation method that is called by XCTest in Xcode 12.5 when a test case is
+  /// skipped.
+  @objc(_XCTestObservationInternal)
+  protocol _XCTestObservationInternal {
+    func testCase(
+      _ testCase: XCTestCase,
+      wasSkippedWithDescription description: String,
+      sourceCodeContext: XCTSourceCodeContext?)
+  }
+
+  extension BazelXMLTestObserver: _XCTestObservationInternal {
+    public func testCase(
+      _ testCase: XCTestCase,
+      wasSkippedWithDescription description: String,
+      sourceCodeContext: XCTSourceCodeContext?
+    ) {
+      self.testCase(
+        testCase,
+        didRecordSkipWithDescription: description,
+        sourceCodeContext: sourceCodeContext)
+    }
+  }
+
+  /// Declares the observation method that is called by XCTest in Xcode 13 and later when a test
+  /// case is skipped.
+  @objc(_XCTestObservationPrivate)
+  protocol _XCTestObservationPrivate {
+    func testCase(
+      _ testCase: XCTestCase,
+      didRecordSkipWithDescription description: String,
+      sourceCodeContext: XCTSourceCodeContext?)
+  }
+
+  extension BazelXMLTestObserver: _XCTestObservationPrivate {
+    public func testCase(
+      _ testCase: XCTestCase,
+      didRecordSkipWithDescription description: String,
+      sourceCodeContext: XCTSourceCodeContext?
+    ) {
+      writeLine(#"<skipped message="\#(xmlEscaping: description)"/>"#)
+    }
+  }
+#endif

tools/test_observer/BazelXMLTestObserverRegistration.swift

@@ -0,0 +1,31 @@
+// Copyright 2022 The Bazel Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#if canImport(ObjectiveC)
+  import Foundation
+  import XCTest
+
+  /// The principal class in an XCTest bundle on Darwin-based platforms, which registers the
+  /// XML-generating observer with the XCTest observation center when the bundle is loaded.
+  @objc(BazelXMLTestObserverRegistration)
+  public final class BazelXMLTestObserverRegistration: NSObject {
+    @objc public override init() {
+      super.init()
+
+      if let observer = BazelXMLTestObserver.default {
+        XCTestObservationCenter.shared.addTestObserver(observer)
+      }
+    }
+  }
+#endif