Change 2/3 to add a new tool that handles pruning the LLDB module cac…

…he of any modules that are found in a metadata file that is intended to be created by a bazel aspect that collects explicit module build outputs. This tool is needed to avoid a crash with LLDB that occurs when the LLDB module cache contains a module that is also generated in a build as an explicit module.

This change handles reading the contents of the implicit module cache and map the name of each module to the absolute path of that module.

PiperOrigin-RevId: 378402314

src/tools/module_cache_pruner/ImplicitModules.swift

@@ -0,0 +1,125 @@
+// Copyright 2021 The Tulsi Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import Foundation
+import os
+
+private let fileManager = FileManager.default
+
+/// Reads the contents of the module cache and returns a dictionary that maps the name of a module
+/// to all pcm files in the module cache for that module.
+func getImplicitModules(moduleCacheURL: URL) -> [String: [URL]] {
+  let implicitModules = getModulesInModuleCache(moduleCacheURL)
+  return mapModuleNamesToModuleURLs(implicitModules)
+}
+
+/// Converts a list of precompiled module URLs into a dictionary that maps module names to the
+/// locations of pcm files for that module.
+private func mapModuleNamesToModuleURLs(_ moduleURLs: [URL]) -> [String: [URL]] {
+  var moduleNameToURLs: [String: [URL]] = [:]
+
+  for url in moduleURLs {
+    if let moduleName = moduleNameForImplicitPrecompiledModule(at: url) {
+      var urlsForModule = moduleNameToURLs[moduleName, default: []]
+      urlsForModule.append(url)
+      moduleNameToURLs[moduleName] = urlsForModule
+    }
+  }
+
+  return moduleNameToURLs
+}
+
+/// Reads the contents of the module cache and returns a list of paths to all the precompiled
+/// files in URL form.
+private func getModulesInModuleCache(_ moduleCacheURL: URL) -> [URL] {
+  let subdirectories = getDirectoriesInModuleCacheRoot(moduleCacheURL)
+  var moduleURLs = [URL]()
+
+  let moduleURLsWriteQueue = DispatchQueue(label: "module-cache-urls")
+  let directoryEnumeratingDispatchGroup = DispatchGroup()
+
+  for subdirectory in subdirectories {
+    directoryEnumeratingDispatchGroup.enter()
+
+    DispatchQueue.global(qos: .default).async {
+      let modulesInSubdirectory = getModulesInModuleCacheSubdirectory(subdirectory)
+      moduleURLsWriteQueue.async {
+        moduleURLs.append(contentsOf: modulesInSubdirectory)
+        directoryEnumeratingDispatchGroup.leave()
+      }
+    }
+  }
+
+  directoryEnumeratingDispatchGroup.wait()
+  return moduleURLs
+}
+
+/// Returns the directories in the root of the module cache directory. Precompiled modules are not
+/// found in the root of the module cache directory, they are only found within the subdirectories
+/// that are returned by this function.
+///
+/// ModuleCache.noindex/
+///   |- <HashA>/
+///   |- ...
+///   |- <HashZ>/
+///   |- <ModuleName-HashAA>.swiftmodule
+///   |- ...
+///   |- <ModuleName-HashZZ>.swiftmodule
+private func getDirectoriesInModuleCacheRoot(_ moduleCacheURL: URL) -> [URL] {
+  do {
+    let contents = try fileManager.contentsOfDirectory(
+      at: moduleCacheURL, includingPropertiesForKeys: nil, options: [])
+    return contents.filter { $0.hasDirectoryPath }
+  } catch {
+    os_log(
+      "Failed to read contents of module cache root at %@: %@", log: logger, type: .error,
+      moduleCacheURL.absoluteString, error.localizedDescription)
+    return []
+  }
+}
+
+/// Returns the precompiled module files found in the given subdirectory. Subdirectories will
+/// contain precompiled modules and temporary timestamps.
+///
+/// ModuleCache.noindex/
+///   |- <HashA>/
+///     |- <ModuleName-HashAA>.pcm
+///     |- ...
+///     |- <ModuleName-HashAZ>.pcm
+///   |- ...
+///   |- <HashZ>/
+///     |- <ModuleName-HashZA>.pcm
+///     |- ...
+///     |- <ModuleName-HashZZ>.pcm
+private func getModulesInModuleCacheSubdirectory(
+  _ directoryURL: URL
+) -> [URL] {
+  do {
+    let contents = try fileManager.contentsOfDirectory(
+      at: directoryURL, includingPropertiesForKeys: nil, options: [])
+    return contents.filter { !$0.hasDirectoryPath && $0.pathExtension == "pcm" }
+  } catch {
+    os_log(
+      "Failed to read contents of module cache subdirectory at %@: %@", log: logger, type: .error,
+      directoryURL.absoluteString, error.localizedDescription)
+    return []
+  }
+}
+
+/// Extracts the module name from an implicit module in LLDB's module cache. Implicit Modules are of
+/// the form "<ModuleName-HashAA>.pcm" e.g. "Foundation-3DFYNEBRQSXST.pcm".
+private func moduleNameForImplicitPrecompiledModule(at moduleURL: URL) -> String? {
+  let filename = moduleURL.lastPathComponent
+  return filename.components(separatedBy: "-").first
+}

src/tools/module_cache_pruner/Logger.swift

@@ -0,0 +1,17 @@
+// Copyright 2021 The Tulsi Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import os
+
+public let logger = OSLog(subsystem: "com.google.ModuleCachePruner", category: "default")

src/tools/module_cache_pruner/tests/FakeModuleCache.swift

@@ -0,0 +1,47 @@
+// Copyright 2021 The Tulsi Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import Foundation
+
+/// Creates a complete directory tree in a temporary directory with the given structure.
+func createFakeModuleCache(with contents: DirectoryStructure) -> URL? {
+  guard let temporaryDirectory = createTemporaryDirectory() else {
+    return nil
+  }
+
+  do {
+    try createDirectoryStructure(temporaryDirectory, withContents: contents)
+  } catch {
+    try? FileManager.default.removeItem(at: temporaryDirectory)
+    return nil
+  }
+  return temporaryDirectory
+}
+
+/// Convenience function that abstracts away the need to provide the exact directory tree structure.
+/// Instead, this function accepts FakeModule objects labeled as either swift or clang modules and
+/// computes the directory structure.
+func createFakeModuleCache(
+  withSwiftModules swiftModules: [FakeModule],
+  andClangModules clangModulesByDirectory: [String: [FakeModule]]
+) -> URL? {
+  var directories = [String: DirectoryStructure]()
+  for (directory, clangModules) in clangModulesByDirectory {
+    directories[directory] = DirectoryStructure(files: clangModules.map { $0.implicitFilename })
+  }
+  let files = swiftModules.map { $0.swiftName }
+
+  let contents = DirectoryStructure(files: files, directories: directories)
+  return createFakeModuleCache(with: contents)
+}

src/tools/module_cache_pruner/tests/FileUtils.swift

@@ -16,6 +16,57 @@ import Foundation
 
 private let fileManager = FileManager.default
 
+struct DirectoryStructure {
+  var files: [String]?
+  var directories: [String: DirectoryStructure]?
+}
+
+/// Creates a directory and any subdirectories + dummy files at the given location on the file
+/// system.
+func createDirectoryStructure(_ directory: URL, withContents contents: DirectoryStructure) throws {
+  if let files = contents.files {
+    for filename in files {
+      let filepath = directory.appendingPathComponent(filename)
+      try "".write(to: filepath, atomically: true, encoding: .utf8)
+    }
+  }
+
+  if let directories = contents.directories {
+    for (dirname, contents) in directories {
+      let subdirectory = directory.appendingPathComponent(dirname)
+      try fileManager.createDirectory(at: subdirectory, withIntermediateDirectories: true)
+      try createDirectoryStructure(subdirectory, withContents: contents)
+    }
+  }
+}
+
+/// Creates a directory with a random name in the macOS temporary directory.
+func createTemporaryDirectory() -> URL? {
+  let osTemporaryDirectory = URL(
+    fileURLWithPath: NSTemporaryDirectory(),
+    isDirectory: true)
+
+  guard
+    let temporaryDirectory = try? fileManager.url(
+      for: .itemReplacementDirectory,
+      in: .userDomainMask,
+      appropriateFor: osTemporaryDirectory,
+      create: true)
+  else {
+    return nil
+  }
+
+  // The macOS temporary directory is located under the `/var` root directory. `/var` is a symlink
+  // to `/private/var` which can cause problems when testing for equal values since equality will
+  // depend on whether prod code resolves the realpath or leaves it as the symlink. To avoid issues,
+  // return the realpath so test and prod code will only ever handle one version of the path.
+  guard let temporaryDirectoryRealPath = realpath(temporaryDirectory.path, nil) else {
+    return nil
+  }
+
+  return URL(fileURLWithPath: String(cString: temporaryDirectoryRealPath), isDirectory: true)
+}
+
 /// Returns a URL to a location for a JSON file with a random filename under the macOS temporary
 /// directory. This function does not create a file at that location.
 func getTemporaryJSONFileURL() -> URL {

src/tools/module_cache_pruner/tests/ImplicitModulesTest.swift

@@ -0,0 +1,80 @@
+// Copyright 2021 The Tulsi Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import Foundation
+import XCTest
+
+@testable import ModuleCachePruner
+
+/// Converts a Dictionary that uses Arrays for the values into a nearly identical Dictionary that
+/// instead uses Sets for the values. This allows us to test for Dictionary equality without
+/// needing to worry about the ordering in the values.
+func convertArrayValuesToSetValues(_ input: [String: [URL]]) -> [String: Set<URL>] {
+  return Dictionary(uniqueKeysWithValues: input.map { ($0, Set($1)) })
+}
+
+class ImplicitModuleTests: XCTestCase {
+  let modules = FakeModules()
+  var fakeModuleCacheURL: URL?
+
+  override func tearDown() {
+    if let moduleCacheURL = fakeModuleCacheURL {
+      try? FileManager.default.removeItem(at: moduleCacheURL)
+    }
+  }
+
+  func testMappingModulesInModuleCache() {
+    guard
+      let moduleCacheURL = createFakeModuleCache(
+        withSwiftModules: [
+          modules.system.foundation, modules.system.coreFoundation, modules.system.darwin,
+        ],
+        andClangModules: [
+          "DirectoryHash1": [
+            modules.user.buttonsLib, modules.user.buttonsIdentity, modules.user.buttonsModel,
+          ],
+          "DirectoryHash2": [
+            modules.user.buttonsLib, modules.user.buttonsIdentity, modules.user.buttonsModel,
+          ],
+        ])
+    else {
+      XCTFail("Failed to create fake module cache required for test.")
+      return
+    }
+
+    fakeModuleCacheURL = moduleCacheURL
+
+    let subdirectory1 = moduleCacheURL.appendingPathComponent("DirectoryHash1")
+    let subdirectory2 = moduleCacheURL.appendingPathComponent("DirectoryHash2")
+    let expectedImplicitModuleMapping = [
+      modules.user.buttonsLib.name: [
+        subdirectory1.appendingPathComponent(modules.user.buttonsLib.implicitFilename),
+        subdirectory2.appendingPathComponent(modules.user.buttonsLib.implicitFilename),
+      ],
+      modules.user.buttonsIdentity.name: [
+        subdirectory1.appendingPathComponent(modules.user.buttonsIdentity.implicitFilename),
+        subdirectory2.appendingPathComponent(modules.user.buttonsIdentity.implicitFilename),
+      ],
+      modules.user.buttonsModel.name: [
+        subdirectory1.appendingPathComponent(modules.user.buttonsModel.implicitFilename),
+        subdirectory2.appendingPathComponent(modules.user.buttonsModel.implicitFilename),
+      ],
+    ]
+
+    let actualImplicitModuleMapping = getImplicitModules(moduleCacheURL: moduleCacheURL)
+    XCTAssertEqual(
+      convertArrayValuesToSetValues(actualImplicitModuleMapping),
+      convertArrayValuesToSetValues(expectedImplicitModuleMapping))
+  }
+}