From e8b8604e486e06a53c68ef1ca311f651df2ab2e6 Mon Sep 17 00:00:00 2001 From: "Jesse L. Zamora" Date: Sun, 5 Jan 2025 23:07:35 -0500 Subject: [PATCH 01/15] Update all package dependencies for Ubuntu 24.04 host --- Package.resolved | 71 +++++++++++++++++++++++++++++++++--------------- 1 file changed, 49 insertions(+), 22 deletions(-) diff --git a/Package.resolved b/Package.resolved index c18045f..6b46724 100644 --- a/Package.resolved +++ b/Package.resolved @@ -5,8 +5,17 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/swift-server/async-http-client.git", "state" : { - "revision" : "16f7e62c08c6969899ce6cc277041e868364e5cf", - "version" : "1.19.0" + "revision" : "2119f0d9cc1b334e25447fe43d3693c0e60e6234", + "version" : "1.24.0" + } + }, + { + "identity" : "swift-algorithms", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-algorithms.git", + "state" : { + "revision" : "f6919dfc309e7f1b56224378b11e28bab5bccc42", + "version" : "1.2.0" } }, { @@ -18,6 +27,15 @@ "version" : "1.5.0" } }, + { + "identity" : "swift-asn1", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-asn1.git", + "state" : { + "revision" : "7faebca1ea4f9aaf0cda1cef7c43aecd2311ddf6", + "version" : "1.3.0" + } + }, { "identity" : "swift-async-algorithms", "kind" : "remoteSourceControl", @@ -41,8 +59,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-collections.git", "state" : { - "revision" : "9bf03ff58ce34478e66aaee630e491823326fd06", - "version" : "1.1.3" + "revision" : "671108c96644956dddcd89dd59c203dcdb36cec7", + "version" : "1.1.4" } }, { @@ -50,8 +68,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-crypto.git", "state" : { - "revision" : "b51f1d6845b353a2121de1c6a670738ec33561a6", - "version" : "3.1.0" + "revision" : "ff0f781cf7c6a22d52957e50b104f5768b50c779", + "version" : "3.10.0" } }, { @@ -59,8 +77,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-http-types", "state" : { - "revision" : "12358d55a3824bd5fed310b999ea8cf83a9a1a65", - "version" : "1.0.3" + "revision" : "ef18d829e8b92d731ad27bb81583edd2094d1ce3", + "version" : "1.3.1" } }, { @@ -68,8 +86,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-log.git", "state" : { - "revision" : "9cb486020ebf03bfa5b5df985387a14a98744537", - "version" : "1.6.1" + "revision" : "96a2f8a0fa41e9e09af4585e2724c4e825410b91", + "version" : "1.6.2" } }, { @@ -77,8 +95,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-nio.git", "state" : { - "revision" : "4c4453b489cf76e6b3b0f300aba663eb78182fad", - "version" : "2.70.0" + "revision" : "dca6594f65308c761a9c409e09fbf35f48d50d34", + "version" : "2.77.0" } }, { @@ -86,8 +104,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-nio-extras.git", "state" : { - "revision" : "363da63c1966405764f380c627409b2f9d9e710b", - "version" : "1.21.0" + "revision" : "2e9746cfc57554f70b650b021b6ae4738abef3e6", + "version" : "1.24.1" } }, { @@ -95,8 +113,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-nio-http2.git", "state" : { - "revision" : "0904bf0feb5122b7e5c3f15db7df0eabe623dd87", - "version" : "1.30.0" + "revision" : "170f4ca06b6a9c57b811293cebcb96e81b661310", + "version" : "1.35.0" } }, { @@ -104,8 +122,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-nio-ssl.git", "state" : { - "revision" : "4fb7ead803e38949eb1d6fabb849206a72c580f3", - "version" : "2.23.0" + "revision" : "c7e95421334b1068490b5d41314a50e70bab23d1", + "version" : "2.29.0" } }, { @@ -113,8 +131,17 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-nio-transport-services.git", "state" : { - "revision" : "e7403c35ca6bb539a7ca353b91cc2d8ec0362d58", - "version" : "1.19.0" + "revision" : "bbd5e63cf949b7db0c9edaf7a21e141c52afe214", + "version" : "1.23.0" + } + }, + { + "identity" : "swift-numerics", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-numerics.git", + "state" : { + "revision" : "0a5bc04095a675662cf24757cc0640aa2204253b", + "version" : "1.0.2" } }, { @@ -122,8 +149,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-system", "state" : { - "revision" : "d2ba781702a1d8285419c15ee62fd734a9437ff5", - "version" : "1.3.2" + "revision" : "c8a44d836fe7913603e246acab7c528c2e780168", + "version" : "1.4.0" } } ], From b4fb926e96e85a1694dd101abf9c8873db2ee4e5 Mon Sep 17 00:00:00 2001 From: "Jesse L. Zamora" Date: Sun, 5 Jan 2025 23:27:04 -0500 Subject: [PATCH 02/15] Cleanup Linux host toolchain, remove lldb --- .../Generator/SwiftSDKGenerator+Unpack.swift | 33 ++++++++++++------- .../SwiftSDKRecipes/WebAssemblyRecipe.swift | 2 +- 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Unpack.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Unpack.swift index 5336c6d..0974344 100644 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Unpack.swift +++ b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Unpack.swift @@ -14,7 +14,7 @@ import Helpers import struct SystemPackage.FilePath -let unusedDarwinPlatforms = [ +let unusedTargetPlatforms = [ "appletvos", "appletvsimulator", "embedded", @@ -34,11 +34,13 @@ let unusedHostBinaries = [ "swift-format", "swift-package", "swift-package-collection", + "lldb*", ] let unusedHostLibraries = [ "sourcekitd.framework", "libsourcekitdInProc.so", + "liblldb.so*", ] extension SwiftSDKGenerator { @@ -49,23 +51,32 @@ extension SwiftSDKGenerator { try self.createDirectoryIfNeeded(at: pathsConfiguration.toolchainDirPath) let excludes = - unusedDarwinPlatforms.map { "--exclude usr/lib/swift/\($0)" } + - unusedDarwinPlatforms.map { "--exclude usr/lib/swift_static/\($0)" } + + unusedTargetPlatforms.map { "--exclude usr/lib/swift/\($0)" } + + unusedTargetPlatforms.map { "--exclude usr/lib/swift_static/\($0)" } + unusedHostBinaries.map { "--exclude usr/bin/\($0)" } + unusedHostLibraries.map { "--exclude usr/lib/\($0)" } - try await Shell.run( - #""" - tar -x --to-stdout -f \#(hostSwiftPackagePath) \*.pkg/Payload | - tar -C "\#(pathsConfiguration.toolchainDirPath)" -x \#(excludes.joined(separator: " ")) --include usr - """#, - shouldLogCommands: isVerbose - ) + if hostSwiftPackagePath.string.contains("tar.gz") { + try await Shell.run( + #""" + tar -xzf \#(hostSwiftPackagePath) -C "\#(pathsConfiguration.toolchainDirPath)" -x \#(excludes.joined(separator: " ")) --strip-components=1 + """#, + shouldLogCommands: isVerbose + ) + } else { + try await Shell.run( + #""" + tar -x --to-stdout -f \#(hostSwiftPackagePath) \*.pkg/Payload | + tar -C "\#(pathsConfiguration.toolchainDirPath)" -x \#(excludes.joined(separator: " ")) --include usr + """#, + shouldLogCommands: isVerbose + ) + } } func removeToolchainComponents( _ packagePath: FilePath, - platforms: [String] = unusedDarwinPlatforms, + platforms: [String] = unusedTargetPlatforms, libraries: [String] = unusedHostLibraries, binaries: [String] = unusedHostBinaries ) async throws { diff --git a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/WebAssemblyRecipe.swift b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/WebAssemblyRecipe.swift index 3385866..e935546 100644 --- a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/WebAssemblyRecipe.swift +++ b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/WebAssemblyRecipe.swift @@ -115,7 +115,7 @@ public struct WebAssemblyRecipe: SwiftSDKRecipe { }() try await generator.removeToolchainComponents( pathsConfiguration.toolchainDirPath, - platforms: unusedDarwinPlatforms + ["embedded"], + platforms: unusedTargetPlatforms, libraries: unusedHostLibraries + liblldbNames, binaries: unusedHostBinaries + ["lldb", "lldb-argdumper", "lldb-server"] ) From 0e05b69266dd17a092b66bf41c1ba2b7b2a63369 Mon Sep 17 00:00:00 2001 From: "Jesse L. Zamora" Date: Mon, 6 Jan 2025 11:29:26 -0500 Subject: [PATCH 03/15] Add note about using amazonlinux2 for host toolchain --- .../Artifacts/DownloadableArtifacts.swift | 34 +++++++++++++------ 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/Sources/SwiftSDKGenerator/Artifacts/DownloadableArtifacts.swift b/Sources/SwiftSDKGenerator/Artifacts/DownloadableArtifacts.swift index beee4b1..c851a0f 100644 --- a/Sources/SwiftSDKGenerator/Artifacts/DownloadableArtifacts.swift +++ b/Sources/SwiftSDKGenerator/Artifacts/DownloadableArtifacts.swift @@ -51,16 +51,30 @@ struct DownloadableArtifacts: Sendable { self.versions = versions self.paths = paths - self.hostSwift = .init( - remoteURL: versions.swiftDownloadURL( - subdirectory: "xcode", - platform: "osx", - fileExtension: "pkg" - ), - localPath: paths.artifactsCachePath - .appending("host_swift_\(versions.swiftVersion)_\(hostTriple.triple).pkg"), - isPrebuilt: true - ) + if hostTriple.os == .linux { + // Amazon Linux 2 is chosen for its best compatibility with all Swift-supported Linux hosts + self.hostSwift = .init( + remoteURL: versions.swiftDownloadURL( + subdirectory: "amazonlinux2", + platform: "amazonlinux2", + fileExtension: "tar.gz" + ), + localPath: paths.artifactsCachePath + .appending("host_swift_\(versions.swiftVersion)_\(hostTriple.triple).tar.gz"), + isPrebuilt: true + ) + } else { + self.hostSwift = .init( + remoteURL: versions.swiftDownloadURL( + subdirectory: "xcode", + platform: "osx", + fileExtension: "pkg" + ), + localPath: paths.artifactsCachePath + .appending("host_swift_\(versions.swiftVersion)_\(hostTriple.triple).pkg"), + isPrebuilt: true + ) + } self.hostLLVM = .init( remoteURL: URL( From 63482faf6f89a75f023ba1503a340485205e3f38 Mon Sep 17 00:00:00 2001 From: "Jesse L. Zamora" Date: Tue, 7 Jan 2025 10:32:09 -0500 Subject: [PATCH 04/15] Update README.md version matrix to add new supported hosts and targets --- README.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index af6ac89..1a031a1 100644 --- a/README.md +++ b/README.md @@ -37,13 +37,16 @@ Linux distributions officially supported by the Swift project. | -: | :- | :- | | macOS (arm64) | ✅ macOS 13.0+ | ❌ | | macOS (x86_64) | ✅ macOS 13.0+[^1] | ❌ | -| Ubuntu | ⚠️ (WIP) | ✅ 20.04 / 22.04 | -| RHEL | ⚠️ (WIP) | ✅ UBI 9 | - +| Ubuntu | ✅ 20.04+ | ✅ 20.04 / 22.04 | +| RHEL | ✅ Fedora 39[^2], UBI 9 | ✅ UBI 9 | +| Amazon Linux 2 | ✅ Supported | ✅ Supported[^3] | +| Debian 12 | ✅ Supported[^2] | ✅ Supported[^2][^3] | [^1]: Since LLVM project doesn't provide pre-built binaries of `lld` for macOS on x86_64, it will be automatically built from sources by the generator, which will increase its run by at least 15 minutes on recent hardware. You will also need CMake and Ninja preinstalled (e.g. via `brew install cmake ninja`). +[^2]: These distributions are only supported by Swift 5.10.1 and later on both the host and target. +[^3]: These versions are technically supported but require custom commands and a Docker container to build the Swift SDK, as the generator will not download dependencies for these distributions automatically. See [issue 138](https://github.com/swiftlang/swift-sdk-generator/issues/138). ## How to use it From f6e812ba33fd2017a1f9d976a63824a189185f81 Mon Sep 17 00:00:00 2001 From: "Jesse L. Zamora" Date: Wed, 8 Jan 2025 00:34:05 -0500 Subject: [PATCH 05/15] Improve logging when extracting deb files --- .../Generator/SwiftSDKGenerator+Download.swift | 1 + Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator.swift | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift index b9f7124..85e0507 100644 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift +++ b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift @@ -114,6 +114,7 @@ extension SwiftSDKGenerator { report(downloadedFiles: downloadedFiles) for fileName in urls.map(\.lastPathComponent) { + print("Extracting \(fileName)...") try await fs.unpack(file: tmpDir.appending(fileName), into: sdkDirPath) } } diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator.swift index 82debf3..04c7155 100644 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator.swift +++ b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator.swift @@ -253,7 +253,9 @@ public actor SwiftSDKGenerator { let isVerbose = self.isVerbose try await self.inTemporaryDirectory { _, tmp in try await Shell.run(#"cd "\#(tmp)" && ar -x "\#(debFile)""#, shouldLogCommands: isVerbose) - try await print(Shell.readStdout("ls \(tmp)")) + if isVerbose { + try await print(Shell.readStdout("ls \(tmp)")) + } try await Shell.run( #"tar -C "\#(directoryPath)" -xf "\#(tmp)"/data.tar.*"#, From b9822fc4023e708d74c93168697388ec2852e0d4 Mon Sep 17 00:00:00 2001 From: "Jesse L. Zamora" Date: Wed, 8 Jan 2025 17:49:47 -0500 Subject: [PATCH 06/15] Resolve errors when generating Swift SDK for Linux host - No need to fetch llvm or prepare lld since the host toolchain for Linux already contains it. - Only create swift_static/clang symlink if it does not exist. --- .../Generator/SwiftSDKGenerator+Fixup.swift | 12 ++++++++---- .../SwiftSDKRecipes/LinuxRecipe.swift | 8 +++++--- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Fixup.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Fixup.swift index fd195b1..4db44f1 100644 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Fixup.swift +++ b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Fixup.swift @@ -49,9 +49,13 @@ extension SwiftSDKGenerator { } func symlinkClangHeaders() throws { - try self.createSymlink( - at: self.pathsConfiguration.toolchainDirPath.appending("usr/lib/swift_static/clang"), - pointingTo: "../swift/clang" - ) + let swiftStaticClangPath = self.pathsConfiguration.toolchainDirPath.appending("usr/lib/swift_static/clang") + if !doesFileExist(at: swiftStaticClangPath) { + logGenerationStep("Symlinking clang headers...") + try self.createSymlink( + at: self.pathsConfiguration.toolchainDirPath.appending("usr/lib/swift_static/clang"), + pointingTo: "../swift/clang" + ) + } } } diff --git a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift index b8fe5b3..a9f95e6 100644 --- a/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift +++ b/Sources/SwiftSDKGenerator/SwiftSDKRecipes/LinuxRecipe.swift @@ -164,7 +164,9 @@ public struct LinuxRecipe: SwiftSDKRecipe { func itemsToDownload(from artifacts: DownloadableArtifacts) -> [DownloadableArtifacts.Item] { var items: [DownloadableArtifacts.Item] = [] - if self.hostSwiftSource != .preinstalled && !self.versionsConfiguration.swiftVersion.hasPrefix("6.0") { + if self.hostSwiftSource != .preinstalled + && self.mainHostTriple.os != .linux + && !self.versionsConfiguration.swiftVersion.hasPrefix("6.0") { items.append(artifacts.hostLLVM) } @@ -279,12 +281,12 @@ public struct LinuxRecipe: SwiftSDKRecipe { try await generator.fixAbsoluteSymlinks(sdkDirPath: sdkDirPath) if self.hostSwiftSource != .preinstalled { - if !self.versionsConfiguration.swiftVersion.hasPrefix("6.0") { + if self.mainHostTriple.os != .linux && !self.versionsConfiguration.swiftVersion.hasPrefix("6.0") { try await generator.prepareLLDLinker(engine, llvmArtifact: downloadableArtifacts.hostLLVM) } if self.versionsConfiguration.swiftVersion.hasPrefix("5.9") || - self.versionsConfiguration.swiftVersion .hasPrefix("5.10") { + self.versionsConfiguration.swiftVersion.hasPrefix("5.10") { try await generator.symlinkClangHeaders() } From 2e6bd289a115989123a2d32ff102408c378953ea Mon Sep 17 00:00:00 2001 From: "Jesse L. Zamora" Date: Wed, 8 Jan 2025 17:51:38 -0500 Subject: [PATCH 07/15] Use Tests/.build directory for EndToEndTests instead of tmp directory - This helps improve the time it takes to run the EndToEndTests since the generator does not need to be rebuild for each test. --- .gitignore | 2 +- Tests/SwiftSDKGeneratorTests/EndToEndTests.swift | 11 +++-------- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index 8e931e6..98deb12 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ .DS_Store -/.build +.build* /.index-build /Packages /*.xcodeproj diff --git a/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift b/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift index 49951eb..12761ff 100644 --- a/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift +++ b/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift @@ -49,11 +49,8 @@ extension FileManager { } } -// Building an SDK requires running the sdk-generator with `swift run swift-sdk-generator`. -// This takes a lock on `.build`, but if the tests are being run by `swift test` the outer Swift Package Manager -// instance will already hold this lock, causing the test to deadlock. We can work around this by giving -// the `swift run swift-sdk-generator` instance its own scratch directory. -func buildSDK(_ logger: Logger, scratchPath: String, withArguments runArguments: String) async throws -> String { +// This will build and run the generator in a separate .build directory from the main one +func buildSDK(_ logger: Logger, scratchPath: String = ".build", withArguments runArguments: String) async throws -> String { var logger = logger logger[metadataKey: "runArguments"] = "\"\(runArguments)\"" logger[metadataKey: "scratchPath"] = "\(scratchPath)" @@ -230,9 +227,7 @@ func buildTestcases(config: SDKConfiguration) async throws { } } - let bundleName = try await FileManager.default.withTemporaryDirectory(logger: logger) { tempDir in - try await buildSDK(logger, scratchPath: tempDir.path, withArguments: config.sdkGeneratorArguments) - } + let bundleName = try await buildSDK(logger, withArguments: config.sdkGeneratorArguments) logger.info("Built SDK") From 2b7d3e394f7323a3bdfc8cbca90e559859c64eb8 Mon Sep 17 00:00:00 2001 From: "Jesse L. Zamora" Date: Wed, 8 Jan 2025 17:52:46 -0500 Subject: [PATCH 08/15] Cleanup installed swift-sdk after running EndToEnd test - The swift-sdk is kept if an end-to-end test fails to allow further debugging. --- Tests/SwiftSDKGeneratorTests/EndToEndTests.swift | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift b/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift index 12761ff..3f65772 100644 --- a/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift +++ b/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift @@ -236,6 +236,10 @@ func buildTestcases(config: SDKConfiguration) async throws { try await buildTestcase(logger, testcase: testcase, bundleName: bundleName, tempDir: tempDir) } } + + // Cleanup + logger.info("Removing SDK to cleanup...") + try await Shell.run("swift experimental-sdk remove \(bundleName)") } final class Swift59_UbuntuEndToEndTests: XCTestCase { From cc05d6feaffbf99a5474e26f0162a7e237bdde50 Mon Sep 17 00:00:00 2001 From: "Jesse L. Zamora" Date: Wed, 8 Jan 2025 17:56:02 -0500 Subject: [PATCH 09/15] Support testing Swift SDKs for Linux from EndToEnd tests - If SWIFT_SDK_GENERATOR_TEST_LINUX_SWIFT_SDKS is set, the swift-sdks are generated for Linux hosts. Each one is then tested against a matrix of Swift docker containers to verify each distribution works with it. - If SWIFT_SDK_GENERATOR_TEST_LINUX_SWIFT_SDKS is NOT set, but the host is Linux, then a Linux swift-sdk is generated but the suite of docker containers is not used. --- .../EndToEndTests.swift | 86 +++++++++++++++---- 1 file changed, 71 insertions(+), 15 deletions(-) diff --git a/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift b/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift index 3f65772..984e5cd 100644 --- a/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift +++ b/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift @@ -165,11 +165,17 @@ struct SDKConfiguration { return res } + var hostArch: String? { + let triple = try? SwiftSDKGenerator.getCurrentTriple(isVerbose: false) + return triple?.arch?.rawValue + } + var sdkGeneratorArguments: String { return [ "--sdk-name \(bundleName)", withDocker ? "--with-docker" : nil, "--swift-version \(swiftVersion)-RELEASE", + testLinuxSwiftSdks ? "--host \(hostArch!)-unknown-linux-gnu" : nil, "--target \(architecture)-unknown-linux-gnu", "--linux-distribution-name \(linuxDistributionName)" ].compactMap{ $0 }.joined(separator: " ") @@ -184,31 +190,81 @@ func skipSlow() throws { ) } +var testLinuxSwiftSdks: Bool { + ProcessInfo.processInfo.environment.keys.contains("SWIFT_SDK_GENERATOR_TEST_LINUX_SWIFT_SDKS") +} + func buildTestcase(_ logger: Logger, testcase: String, bundleName: String, tempDir: URL) async throws { let testPackageURL = tempDir.appendingPathComponent("swift-sdk-generator-test") let testPackageDir = FilePath(testPackageURL.path) try FileManager.default.createDirectory(atPath: testPackageDir.string, withIntermediateDirectories: true) - logger.info("Creating test project") + logger.info("Creating test project \(testPackageDir)") try await Shell.run("swift package --package-path \(testPackageDir) init --type executable") let main_swift = testPackageURL.appendingPathComponent("Sources/main.swift") try testcase.write(to: main_swift, atomically: true, encoding: .utf8) - logger.info("Building test project") - var buildOutput = try await Shell.readStdout( - "swift build --package-path \(testPackageDir) --experimental-swift-sdk \(bundleName)" - ) - XCTAssertTrue(buildOutput.contains("Build complete!")) - logger.info("Test project built successfully") - - try await Shell.run("rm -rf \(testPackageDir.appending(".build"))") + // This is a workaround for if Swift 6.0 is used as the host toolchain to run the generator. + // We manually set the swift-tools-version to 5.9 to support building our test cases. + logger.info("Updating minimum swift-tools-version in test project...") + let package_swift = testPackageURL.appendingPathComponent("Package.swift") + let text = try String(contentsOf: package_swift, encoding: .utf8) + var lines = text.components(separatedBy: .newlines) + if lines.count > 0 { + lines[0] = "// swift-tools-version: 5.9" + let result = lines.joined(separator: "\r\n") + try result.write(to: package_swift, atomically: true, encoding: .utf8) + } - logger.info("Building test project with static-swift-stdlib") - buildOutput = try await Shell.readStdout( - "swift build --package-path \(testPackageDir) --experimental-swift-sdk \(bundleName) --static-swift-stdlib" - ) - XCTAssertTrue(buildOutput.contains("Build complete!")) - logger.info("Test project built successfully") + var buildOutput = "" + + // If we are testing Linux Swift SDKs, we will run the test cases on a matrix of Docker containers + // that contains each Swift-supported Linux distribution. This way we can validate that each + // distribution is capable of building using the Linux Swift SDK. + if testLinuxSwiftSdks { + let swiftContainerVersions = ["focal", "jammy", "noble", "fedora39", "rhel-ubi9", "amazonlinux2", "bookworm"] + for containerVersion in swiftContainerVersions { + logger.info("Building test project in 6.0-\(containerVersion) container") + buildOutput = try await Shell.readStdout( + """ + docker run --rm -v \(testPackageDir):/src \ + -v $HOME/.swiftpm/swift-sdks:/root/.swiftpm/swift-sdks \ + --workdir /src swift:6.0-\(containerVersion) \ + /bin/bash -c "swift build --scratch-path /root/.build --experimental-swift-sdk \(bundleName)" + """ + ) + XCTAssertTrue(buildOutput.contains("Build complete!")) + logger.info("Test project built successfully") + + logger.info("Building test project in 6.0-\(containerVersion) container with static-swift-stdlib") + buildOutput = try await Shell.readStdout( + """ + docker run --rm -v \(testPackageDir):/src \ + -v $HOME/.swiftpm/swift-sdks:/root/.swiftpm/swift-sdks \ + --workdir /src swift:6.0-\(containerVersion) \ + /bin/bash -c "swift build --scratch-path /root/.build --experimental-swift-sdk \(bundleName) --static-swift-stdlib" + """ + ) + XCTAssertTrue(buildOutput.contains("Build complete!")) + logger.info("Test project built successfully") + } + } else { + logger.info("Building test project") + buildOutput = try await Shell.readStdout( + "swift build --package-path \(testPackageDir) --experimental-swift-sdk \(bundleName)" + ) + XCTAssertTrue(buildOutput.contains("Build complete!")) + logger.info("Test project built successfully") + + try await Shell.run("rm -rf \(testPackageDir.appending(".build"))") + + logger.info("Building test project with static-swift-stdlib") + buildOutput = try await Shell.readStdout( + "swift build --package-path \(testPackageDir) --experimental-swift-sdk \(bundleName) --static-swift-stdlib" + ) + XCTAssertTrue(buildOutput.contains("Build complete!")) + logger.info("Test project built successfully") + } } func buildTestcases(config: SDKConfiguration) async throws { From e53d8791e4864a1c88836ed9e982cde402c5553a Mon Sep 17 00:00:00 2001 From: "Jesse L. Zamora" Date: Wed, 8 Jan 2025 18:35:20 -0500 Subject: [PATCH 10/15] Fix host arch for downloading Linux toolchain - If the host is aarch64, we download the aarch64 Swift toolchain. --- .../SwiftSDKGenerator/Artifacts/DownloadableArtifacts.swift | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Sources/SwiftSDKGenerator/Artifacts/DownloadableArtifacts.swift b/Sources/SwiftSDKGenerator/Artifacts/DownloadableArtifacts.swift index c851a0f..88fb7d5 100644 --- a/Sources/SwiftSDKGenerator/Artifacts/DownloadableArtifacts.swift +++ b/Sources/SwiftSDKGenerator/Artifacts/DownloadableArtifacts.swift @@ -53,10 +53,11 @@ struct DownloadableArtifacts: Sendable { if hostTriple.os == .linux { // Amazon Linux 2 is chosen for its best compatibility with all Swift-supported Linux hosts + let linuxArchSuffix = hostTriple.arch == .aarch64 ? "-\(Triple.Arch.aarch64.linuxConventionName)" : "" self.hostSwift = .init( remoteURL: versions.swiftDownloadURL( - subdirectory: "amazonlinux2", - platform: "amazonlinux2", + subdirectory: "amazonlinux2\(linuxArchSuffix)", + platform: "amazonlinux2\(linuxArchSuffix)", fileExtension: "tar.gz" ), localPath: paths.artifactsCachePath From 2ab3587352d254afe1f8f8bd9a6c9b2011589b3b Mon Sep 17 00:00:00 2001 From: "Jesse L. Zamora" Date: Thu, 9 Jan 2025 09:33:28 -0500 Subject: [PATCH 11/15] Restore Package.resolved to locked deps for CI - If running on a Ubuntu 24.04 host, simply do swift package update locally and do not check in the updated Package.resolved. --- Package.resolved | 71 +++++++++++++++--------------------------------- 1 file changed, 22 insertions(+), 49 deletions(-) diff --git a/Package.resolved b/Package.resolved index 6b46724..c18045f 100644 --- a/Package.resolved +++ b/Package.resolved @@ -5,17 +5,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/swift-server/async-http-client.git", "state" : { - "revision" : "2119f0d9cc1b334e25447fe43d3693c0e60e6234", - "version" : "1.24.0" - } - }, - { - "identity" : "swift-algorithms", - "kind" : "remoteSourceControl", - "location" : "https://github.com/apple/swift-algorithms.git", - "state" : { - "revision" : "f6919dfc309e7f1b56224378b11e28bab5bccc42", - "version" : "1.2.0" + "revision" : "16f7e62c08c6969899ce6cc277041e868364e5cf", + "version" : "1.19.0" } }, { @@ -27,15 +18,6 @@ "version" : "1.5.0" } }, - { - "identity" : "swift-asn1", - "kind" : "remoteSourceControl", - "location" : "https://github.com/apple/swift-asn1.git", - "state" : { - "revision" : "7faebca1ea4f9aaf0cda1cef7c43aecd2311ddf6", - "version" : "1.3.0" - } - }, { "identity" : "swift-async-algorithms", "kind" : "remoteSourceControl", @@ -59,8 +41,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-collections.git", "state" : { - "revision" : "671108c96644956dddcd89dd59c203dcdb36cec7", - "version" : "1.1.4" + "revision" : "9bf03ff58ce34478e66aaee630e491823326fd06", + "version" : "1.1.3" } }, { @@ -68,8 +50,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-crypto.git", "state" : { - "revision" : "ff0f781cf7c6a22d52957e50b104f5768b50c779", - "version" : "3.10.0" + "revision" : "b51f1d6845b353a2121de1c6a670738ec33561a6", + "version" : "3.1.0" } }, { @@ -77,8 +59,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-http-types", "state" : { - "revision" : "ef18d829e8b92d731ad27bb81583edd2094d1ce3", - "version" : "1.3.1" + "revision" : "12358d55a3824bd5fed310b999ea8cf83a9a1a65", + "version" : "1.0.3" } }, { @@ -86,8 +68,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-log.git", "state" : { - "revision" : "96a2f8a0fa41e9e09af4585e2724c4e825410b91", - "version" : "1.6.2" + "revision" : "9cb486020ebf03bfa5b5df985387a14a98744537", + "version" : "1.6.1" } }, { @@ -95,8 +77,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-nio.git", "state" : { - "revision" : "dca6594f65308c761a9c409e09fbf35f48d50d34", - "version" : "2.77.0" + "revision" : "4c4453b489cf76e6b3b0f300aba663eb78182fad", + "version" : "2.70.0" } }, { @@ -104,8 +86,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-nio-extras.git", "state" : { - "revision" : "2e9746cfc57554f70b650b021b6ae4738abef3e6", - "version" : "1.24.1" + "revision" : "363da63c1966405764f380c627409b2f9d9e710b", + "version" : "1.21.0" } }, { @@ -113,8 +95,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-nio-http2.git", "state" : { - "revision" : "170f4ca06b6a9c57b811293cebcb96e81b661310", - "version" : "1.35.0" + "revision" : "0904bf0feb5122b7e5c3f15db7df0eabe623dd87", + "version" : "1.30.0" } }, { @@ -122,8 +104,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-nio-ssl.git", "state" : { - "revision" : "c7e95421334b1068490b5d41314a50e70bab23d1", - "version" : "2.29.0" + "revision" : "4fb7ead803e38949eb1d6fabb849206a72c580f3", + "version" : "2.23.0" } }, { @@ -131,17 +113,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-nio-transport-services.git", "state" : { - "revision" : "bbd5e63cf949b7db0c9edaf7a21e141c52afe214", - "version" : "1.23.0" - } - }, - { - "identity" : "swift-numerics", - "kind" : "remoteSourceControl", - "location" : "https://github.com/apple/swift-numerics.git", - "state" : { - "revision" : "0a5bc04095a675662cf24757cc0640aa2204253b", - "version" : "1.0.2" + "revision" : "e7403c35ca6bb539a7ca353b91cc2d8ec0362d58", + "version" : "1.19.0" } }, { @@ -149,8 +122,8 @@ "kind" : "remoteSourceControl", "location" : "https://github.com/apple/swift-system", "state" : { - "revision" : "c8a44d836fe7913603e246acab7c528c2e780168", - "version" : "1.4.0" + "revision" : "d2ba781702a1d8285419c15ee62fd734a9437ff5", + "version" : "1.3.2" } } ], From e3a4f2612f4340e56c0498494252cab91686e864 Mon Sep 17 00:00:00 2001 From: "Jesse L. Zamora" Date: Thu, 9 Jan 2025 11:32:47 -0500 Subject: [PATCH 12/15] Apply README and EndToEndTests suggestions --- README.md | 4 ++-- Tests/SwiftSDKGeneratorTests/EndToEndTests.swift | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 1a031a1..0e723e9 100644 --- a/README.md +++ b/README.md @@ -45,8 +45,8 @@ Linux distributions officially supported by the Swift project. [^1]: Since LLVM project doesn't provide pre-built binaries of `lld` for macOS on x86_64, it will be automatically built from sources by the generator, which will increase its run by at least 15 minutes on recent hardware. You will also need CMake and Ninja preinstalled (e.g. via `brew install cmake ninja`). -[^2]: These distributions are only supported by Swift 5.10.1 and later on both the host and target. -[^3]: These versions are technically supported but require custom commands and a Docker container to build the Swift SDK, as the generator will not download dependencies for these distributions automatically. See [issue 138](https://github.com/swiftlang/swift-sdk-generator/issues/138). +[^2]: These distributions are only supported by Swift 5.10.1 and later as both host and target platforms. +[^3]: These versions are technically supported but require custom commands and a Docker container to build the Swift SDK, as the generator will not download dependencies for these distributions automatically. See [issue #138](https://github.com/swiftlang/swift-sdk-generator/issues/138). ## How to use it diff --git a/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift b/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift index 984e5cd..8570339 100644 --- a/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift +++ b/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift @@ -175,7 +175,7 @@ struct SDKConfiguration { "--sdk-name \(bundleName)", withDocker ? "--with-docker" : nil, "--swift-version \(swiftVersion)-RELEASE", - testLinuxSwiftSdks ? "--host \(hostArch!)-unknown-linux-gnu" : nil, + testLinuxSwiftSDKs ? "--host \(hostArch!)-unknown-linux-gnu" : nil, "--target \(architecture)-unknown-linux-gnu", "--linux-distribution-name \(linuxDistributionName)" ].compactMap{ $0 }.joined(separator: " ") @@ -190,7 +190,7 @@ func skipSlow() throws { ) } -var testLinuxSwiftSdks: Bool { +var testLinuxSwiftSDKs: Bool { ProcessInfo.processInfo.environment.keys.contains("SWIFT_SDK_GENERATOR_TEST_LINUX_SWIFT_SDKS") } @@ -221,7 +221,7 @@ func buildTestcase(_ logger: Logger, testcase: String, bundleName: String, tempD // If we are testing Linux Swift SDKs, we will run the test cases on a matrix of Docker containers // that contains each Swift-supported Linux distribution. This way we can validate that each // distribution is capable of building using the Linux Swift SDK. - if testLinuxSwiftSdks { + if testLinuxSwiftSDKs { let swiftContainerVersions = ["focal", "jammy", "noble", "fedora39", "rhel-ubi9", "amazonlinux2", "bookworm"] for containerVersion in swiftContainerVersions { logger.info("Building test project in 6.0-\(containerVersion) container") From 19119c1778cbdc0719ff81af4f52ded0caec684f Mon Sep 17 00:00:00 2001 From: "Jesse L. Zamora" Date: Thu, 9 Jan 2025 11:40:19 -0500 Subject: [PATCH 13/15] Revert "Use Tests/.build directory for EndToEndTests instead of tmp directory" This reverts commit 2e6bd289a115989123a2d32ff102408c378953ea. --- .gitignore | 2 +- Tests/SwiftSDKGeneratorTests/EndToEndTests.swift | 11 ++++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 98deb12..8e931e6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ .DS_Store -.build* +/.build /.index-build /Packages /*.xcodeproj diff --git a/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift b/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift index 8570339..2026387 100644 --- a/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift +++ b/Tests/SwiftSDKGeneratorTests/EndToEndTests.swift @@ -49,8 +49,11 @@ extension FileManager { } } -// This will build and run the generator in a separate .build directory from the main one -func buildSDK(_ logger: Logger, scratchPath: String = ".build", withArguments runArguments: String) async throws -> String { +// Building an SDK requires running the sdk-generator with `swift run swift-sdk-generator`. +// This takes a lock on `.build`, but if the tests are being run by `swift test` the outer Swift Package Manager +// instance will already hold this lock, causing the test to deadlock. We can work around this by giving +// the `swift run swift-sdk-generator` instance its own scratch directory. +func buildSDK(_ logger: Logger, scratchPath: String, withArguments runArguments: String) async throws -> String { var logger = logger logger[metadataKey: "runArguments"] = "\"\(runArguments)\"" logger[metadataKey: "scratchPath"] = "\(scratchPath)" @@ -283,7 +286,9 @@ func buildTestcases(config: SDKConfiguration) async throws { } } - let bundleName = try await buildSDK(logger, withArguments: config.sdkGeneratorArguments) + let bundleName = try await FileManager.default.withTemporaryDirectory(logger: logger) { tempDir in + try await buildSDK(logger, scratchPath: tempDir.path, withArguments: config.sdkGeneratorArguments) + } logger.info("Built SDK") From 4a2f69d9330c1188ab914e1607ee68f97a2cf3d9 Mon Sep 17 00:00:00 2001 From: "Jesse L. Zamora" Date: Thu, 16 Jan 2025 11:45:48 -0500 Subject: [PATCH 14/15] Revert "Improve logging when extracting deb files" This reverts commit f6e812ba33fd2017a1f9d976a63824a189185f81. --- .../Generator/SwiftSDKGenerator+Download.swift | 1 - Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator.swift | 4 +--- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift index 85e0507..b9f7124 100644 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift +++ b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Download.swift @@ -114,7 +114,6 @@ extension SwiftSDKGenerator { report(downloadedFiles: downloadedFiles) for fileName in urls.map(\.lastPathComponent) { - print("Extracting \(fileName)...") try await fs.unpack(file: tmpDir.appending(fileName), into: sdkDirPath) } } diff --git a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator.swift b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator.swift index 04c7155..82debf3 100644 --- a/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator.swift +++ b/Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator.swift @@ -253,9 +253,7 @@ public actor SwiftSDKGenerator { let isVerbose = self.isVerbose try await self.inTemporaryDirectory { _, tmp in try await Shell.run(#"cd "\#(tmp)" && ar -x "\#(debFile)""#, shouldLogCommands: isVerbose) - if isVerbose { - try await print(Shell.readStdout("ls \(tmp)")) - } + try await print(Shell.readStdout("ls \(tmp)")) try await Shell.run( #"tar -C "\#(directoryPath)" -xf "\#(tmp)"/data.tar.*"#, From f38e1b4917916757cdbe8b50b2acb5fb030cdc20 Mon Sep 17 00:00:00 2001 From: "Jesse L. Zamora" Date: Thu, 16 Jan 2025 12:21:17 -0500 Subject: [PATCH 15/15] Fix LinuxRecipeTests, add tests cases for Linux vs macOS - On Linux we do not download the host LLVM since it is already included in the toolchain. However, on macOS we may or may not need to download it. --- .../SwiftSDKRecipes/LinuxRecipeTests.swift | 115 ++++++++++++++---- 1 file changed, 93 insertions(+), 22 deletions(-) diff --git a/Tests/SwiftSDKGeneratorTests/SwiftSDKRecipes/LinuxRecipeTests.swift b/Tests/SwiftSDKGeneratorTests/SwiftSDKRecipes/LinuxRecipeTests.swift index 1440a92..75f9737 100644 --- a/Tests/SwiftSDKGeneratorTests/SwiftSDKRecipes/LinuxRecipeTests.swift +++ b/Tests/SwiftSDKGeneratorTests/SwiftSDKRecipes/LinuxRecipeTests.swift @@ -16,6 +16,7 @@ import XCTest final class LinuxRecipeTests: XCTestCase { func createRecipe( + hostTriple: Triple = Triple("x86_64-unknown-linux-gnu"), swiftVersion: String = "6.0", withDocker: Bool = false, fromContainerImage: String? = nil, @@ -25,7 +26,7 @@ final class LinuxRecipeTests: XCTestCase { ) throws -> LinuxRecipe { try LinuxRecipe( targetTriple: Triple("aarch64-unknown-linux-gnu"), - hostTriple: Triple("x86_64-unknown-linux-gnu"), + hostTriple: hostTriple, linuxDistribution: .init(name: .ubuntu, version: "22.04"), swiftVersion: swiftVersion, swiftBranch: nil, lldVersion: "", @@ -85,25 +86,51 @@ final class LinuxRecipeTests: XCTestCase { XCTAssert(toolset.linker == nil) } - func testItemsToDownload() throws { - let testCases = [ + func runItemsToDownloadTestCase( + recipe: LinuxRecipe, includesHostLLVM: Bool, includesTargetSwift: Bool, includesHostSwift: Bool + ) throws { + let pathsConfiguration = PathsConfiguration( + sourceRoot: ".", + artifactID: "my-sdk", + targetTriple: recipe.mainTargetTriple + ) + let downloadableArtifacts = try DownloadableArtifacts( + hostTriple: recipe.mainHostTriple, + targetTriple: recipe.mainTargetTriple, + recipe.versionsConfiguration, + pathsConfiguration + ) + let itemsToDownload = recipe.itemsToDownload(from: downloadableArtifacts) + let foundHostLLVM = itemsToDownload.contains(where: { $0.remoteURL == downloadableArtifacts.hostLLVM.remoteURL }) + let foundTargetSwift = itemsToDownload.contains(where: { $0.remoteURL == downloadableArtifacts.targetSwift.remoteURL }) + let foundHostSwift = itemsToDownload.contains(where: { $0.remoteURL == downloadableArtifacts.hostSwift.remoteURL }) + + // If this is a Linux host, we do not download LLVM + XCTAssertEqual(foundHostLLVM, includesHostLLVM) + XCTAssertEqual(foundTargetSwift, includesTargetSwift) + XCTAssertEqual(foundHostSwift, includesHostSwift) + } + + func testItemsToDownloadForMacOSHost() throws { + let hostTriple = Triple("x86_64-apple-macos") + let testCases: [(recipe: LinuxRecipe, includesHostLLVM: Bool, includesTargetSwift: Bool, includesHostSwift: Bool)] = [ ( // Remote tarballs on Swift < 6.0 - recipe: try createRecipe(swiftVersion: "5.10"), + recipe: try createRecipe(hostTriple: hostTriple, swiftVersion: "5.10"), includesHostLLVM: true, includesTargetSwift: true, includesHostSwift: true ), ( // Remote tarballs on Swift >= 6.0 - recipe: try createRecipe(swiftVersion: "6.0"), + recipe: try createRecipe(hostTriple: hostTriple, swiftVersion: "6.0"), includesHostLLVM: false, includesTargetSwift: true, includesHostSwift: true ), ( // Remote target tarball with preinstalled toolchain - recipe: try createRecipe(swiftVersion: "5.9", includeHostToolchain: false), + recipe: try createRecipe(hostTriple: hostTriple, swiftVersion: "5.9", includeHostToolchain: false), includesHostLLVM: false, includesTargetSwift: true, includesHostSwift: false @@ -111,6 +138,7 @@ final class LinuxRecipeTests: XCTestCase { ( // Local packages with Swift < 6.0 recipe: try createRecipe( + hostTriple: hostTriple, swiftVersion: "5.10", hostSwiftPackagePath: "/path/to/host/swift", targetSwiftPackagePath: "/path/to/target/swift" @@ -122,6 +150,7 @@ final class LinuxRecipeTests: XCTestCase { ( // Local packages with Swift >= 6.0 recipe: try createRecipe( + hostTriple: hostTriple, swiftVersion: "6.0", hostSwiftPackagePath: "/path/to/host/swift", targetSwiftPackagePath: "/path/to/target/swift" @@ -133,25 +162,67 @@ final class LinuxRecipeTests: XCTestCase { ] for testCase in testCases { - let pathsConfiguration = PathsConfiguration( - sourceRoot: ".", - artifactID: "my-sdk", - targetTriple: testCase.recipe.mainTargetTriple + try runItemsToDownloadTestCase( + recipe: testCase.recipe, + includesHostLLVM: testCase.includesHostLLVM, + includesTargetSwift: testCase.includesTargetSwift, + includesHostSwift: testCase.includesHostSwift ) - let downloadableArtifacts = try DownloadableArtifacts( - hostTriple: testCase.recipe.mainHostTriple, - targetTriple: testCase.recipe.mainTargetTriple, - testCase.recipe.versionsConfiguration, - pathsConfiguration + } + } + + func testItemsToDownloadForLinuxHost() throws { + let hostTriple = Triple("x86_64-unknown-linux-gnu") + let testCases = [ + ( + // Remote tarballs on Swift < 6.0 + recipe: try createRecipe(hostTriple: hostTriple, swiftVersion: "5.10"), + includesTargetSwift: true, + includesHostSwift: true + ), + ( + // Remote tarballs on Swift >= 6.0 + recipe: try createRecipe(hostTriple: hostTriple, swiftVersion: "6.0"), + includesTargetSwift: true, + includesHostSwift: true + ), + ( + // Remote target tarball with preinstalled toolchain + recipe: try createRecipe(hostTriple: hostTriple, swiftVersion: "5.9", includeHostToolchain: false), + includesTargetSwift: true, + includesHostSwift: false + ), + ( + // Local packages with Swift < 6.0 + recipe: try createRecipe( + hostTriple: hostTriple, + swiftVersion: "5.10", + hostSwiftPackagePath: "/path/to/host/swift", + targetSwiftPackagePath: "/path/to/target/swift" + ), + includesTargetSwift: false, + includesHostSwift: false + ), + ( + // Local packages with Swift >= 6.0 + recipe: try createRecipe( + hostTriple: hostTriple, + swiftVersion: "6.0", + hostSwiftPackagePath: "/path/to/host/swift", + targetSwiftPackagePath: "/path/to/target/swift" + ), + includesTargetSwift: false, + includesHostSwift: false ) - let itemsToDownload = testCase.recipe.itemsToDownload(from: downloadableArtifacts) - let foundHostLLVM = itemsToDownload.contains(where: { $0.remoteURL == downloadableArtifacts.hostLLVM.remoteURL }) - let foundTargetSwift = itemsToDownload.contains(where: { $0.remoteURL == downloadableArtifacts.targetSwift.remoteURL }) - let foundHostSwift = itemsToDownload.contains(where: { $0.remoteURL == downloadableArtifacts.hostSwift.remoteURL }) + ] - XCTAssertEqual(foundHostLLVM, testCase.includesHostLLVM) - XCTAssertEqual(foundTargetSwift, testCase.includesTargetSwift) - XCTAssertEqual(foundHostSwift, testCase.includesHostSwift) + for testCase in testCases { + try runItemsToDownloadTestCase( + recipe: testCase.recipe, + includesHostLLVM: false, // when host is Linux we do not download LLVM + includesTargetSwift: testCase.includesTargetSwift, + includesHostSwift: testCase.includesHostSwift + ) } }