swift: fix build with the new SDK pattern (#346947)

pkgs/development/compilers/swift/compiler/default.nix

@@ -30,38 +30,29 @@
 # Darwin-specific
 , substituteAll
 , fixDarwinDylibNames
-, runCommandLocal
 , xcbuild
 , cctools # libtool
 , sigtool
 , DarwinTools
-, CoreServices
-, Foundation
-, Combine
-, MacOSX-SDK
-, CLTools_Executables
+, apple-sdk_13
+, darwinMinVersionHook
 }:
 
 let
+  apple-sdk_swift = apple-sdk_13; # Use the SDK that was available when Swift shipped.
+
+  deploymentVersion =
+    if lib.versionOlder (targetPlatform.darwinMinVersion or "0") "10.15" then
+      "10.15"
+    else
+      targetPlatform.darwinMinVersion;
+
   python3 = python3Packages.python.withPackages (p: [ p.setuptools ]); # python 3.12 compat.
 
   inherit (stdenv) hostPlatform targetPlatform;
 
   sources = callPackage ../sources.nix { };
 
-  # Tools invoked by swift at run-time.
-  runtimeDeps = lib.optionals stdenv.hostPlatform.isDarwin [
-    # libtool is used for static linking. This is part of cctools, but adding
-    # that as a build input puts an unwrapped linker in PATH, and breaks
-    # builds. This small derivation exposes just libtool.
-    # NOTE: The same applies to swift-driver, but that is currently always
-    # invoked via the old `swift` / `swiftc`. May change in the future.
-    (runCommandLocal "libtool" { } ''
-      mkdir -p $out/bin
-      ln -s ${cctools}/bin/libtool $out/bin/libtool
-    '')
-  ];
-
   # There are apparently multiple naming conventions on Darwin. Swift uses the
   # xcrun naming convention. See `configure_sdk_darwin` calls in CMake files.
   swiftOs = if targetPlatform.isDarwin
@@ -158,7 +149,9 @@ let
     # NOTE: @prog@ needs to be filled elsewhere.
   };
   swiftWrapper = runCommand "swift-wrapper.sh" wrapperParams ''
-    substituteAll '${../wrapper/wrapper.sh}' "$out"
+    # Make empty to avoid adding the SDK’s modules in the bootstrap wrapper. Otherwise, the SDK conflicts with the
+    # shims the wrapper tries to build.
+    darwinMinVersion="" substituteAll '${../wrapper/wrapper.sh}' "$out"
   '';
   makeSwiftcWrapper = writeShellScriptBin "nix-swift-make-swift-wrapper" ''
     set -euo pipefail
@@ -179,11 +172,20 @@ let
     name = "apple-swift-core";
     dontUnpack = true;
 
+    buildInputs = [ apple-sdk_swift ];
+
     installPhase = ''
       mkdir -p $out/lib/swift
       cp -r \
-        "${MacOSX-SDK}/usr/lib/swift/Swift.swiftmodule" \
-        "${MacOSX-SDK}/usr/lib/swift/libswiftCore.tbd" \
+        "$SDKROOT/usr/lib/swift/Swift.swiftmodule" \
+        "$SDKROOT/usr/lib/swift/CoreFoundation.swiftmodule" \
+        "$SDKROOT/usr/lib/swift/Dispatch.swiftmodule" \
+        "$SDKROOT/usr/lib/swift/ObjectiveC.swiftmodule" \
+        "$SDKROOT/usr/lib/swift/libswiftCore.tbd" \
+        "$SDKROOT/usr/lib/swift/libswiftCoreFoundation.tbd" \
+        "$SDKROOT/usr/lib/swift/libswiftDispatch.tbd" \
+        "$SDKROOT/usr/lib/swift/libswiftFoundation.tbd" \
+        "$SDKROOT/usr/lib/swift/libswiftObjectiveC.tbd" \
         $out/lib/swift/
     '';
   };
@@ -210,6 +212,7 @@ in stdenv.mkDerivation {
       sigtool # codesign
       DarwinTools # sw_vers
       fixDarwinDylibNames
+      cctools.libtool
     ];
 
   buildInputs = [
@@ -222,11 +225,16 @@ in stdenv.mkDerivation {
       libuuid
     ]
     ++ lib.optionals stdenv.hostPlatform.isDarwin [
-      CoreServices
-      Foundation
-      Combine
+      apple-sdk_swift
+      (darwinMinVersionHook deploymentVersion)
     ];
 
+  # Will effectively be `buildInputs` when swift is put in `nativeBuildInputs`.
+  depsTargetTargetPropagated = lib.optionals stdenv.targetPlatform.isDarwin [
+    apple-sdk_swift
+    (darwinMinVersionHook deploymentVersion)
+  ];
+
   # This is a partial reimplementation of our setup hook. Because we reuse
   # the Swift wrapper for the Swift build itself, we need to do some of the
   # same preparation.
@@ -399,9 +407,6 @@ in stdenv.mkDerivation {
     mkdir -p ../build
     cd ../build
     export SWIFT_BUILD_ROOT="$PWD"
-
-    # Most builds set a target, but LLDB doesn't. Harmless on non-Darwin.
-    export MACOSX_DEPLOYMENT_TARGET=10.15
   '';
 
   # These steps are derived from doing a normal build with.
@@ -452,14 +457,10 @@ in stdenv.mkDerivation {
     buildProject llvm llvm-project/llvm
 
     '' + lib.optionalString stdenv.hostPlatform.isDarwin ''
-    # Add appleSwiftCore to the search paths. We can't simply add it to
-    # buildInputs, because it is potentially an older stdlib than the one we're
-    # building. We have to remove it again after the main Swift build, or later
-    # build steps may fail. (Specific case: Concurrency backdeploy uses the
-    # Sendable protocol, which appears to not be present in the macOS 11 SDK.)
+    # Add appleSwiftCore to the search paths. Adding the whole SDK results in build failures.
     OLD_NIX_SWIFTFLAGS_COMPILE="$NIX_SWIFTFLAGS_COMPILE"
     OLD_NIX_LDFLAGS="$NIX_LDFLAGS"
-    export NIX_SWIFTFLAGS_COMPILE+=" -I ${appleSwiftCore}/lib/swift"
+    export NIX_SWIFTFLAGS_COMPILE=" -I ${appleSwiftCore}/lib/swift"
     export NIX_LDFLAGS+=" -L ${appleSwiftCore}/lib/swift"
     '' + ''
 
@@ -491,6 +492,7 @@ in stdenv.mkDerivation {
       -DSWIFT_PATH_TO_STRING_PROCESSING_SOURCE=$SWIFT_SOURCE_ROOT/swift-experimental-string-processing
       -DSWIFT_INSTALL_COMPONENTS=${lib.concatStringsSep ";" swiftInstallComponents}
       -DSWIFT_STDLIB_ENABLE_OBJC_INTEROP=${if stdenv.hostPlatform.isDarwin then "ON" else "OFF"}
+      -DSWIFT_DARWIN_DEPLOYMENT_VERSION_OSX=${deploymentVersion}
     "
     buildProject swift
 
@@ -506,12 +508,6 @@ in stdenv.mkDerivation {
     # which requires a special signature.
     #
     # CMAKE_BUILD_WITH_INSTALL_NAME_DIR ensures we don't use rpath on Darwin.
-    #
-    # NOTE: On Darwin, we only want ncurses in the linker search path, because
-    # headers are part of libsystem. Adding its headers to the search path
-    # causes strange mixing and errors. Note that libedit propagates ncurses,
-    # so we add both manually here, instead of relying on setup hooks.
-    # TODO: Find a better way to prevent this conflict.
     cmakeFlags="
       -GNinja
       -DLLDB_SWIFTC=$SWIFT_BUILD_ROOT/swift/bin/swiftc
@@ -529,11 +525,11 @@ in stdenv.mkDerivation {
       ${lib.optionalString stdenv.hostPlatform.isDarwin ''
       -DLLDB_USE_SYSTEM_DEBUGSERVER=ON
       ''}
-      -DLibEdit_INCLUDE_DIRS=${libedit.dev}/include
-      -DLibEdit_LIBRARIES=${libedit}/lib/libedit${stdenv.hostPlatform.extensions.sharedLibrary}
-      -DCURSES_INCLUDE_DIRS=${if stdenv.hostPlatform.isDarwin then "/var/empty" else ncurses.dev}/include
-      -DCURSES_LIBRARIES=${ncurses}/lib/libncurses${stdenv.hostPlatform.extensions.sharedLibrary}
-      -DPANEL_LIBRARIES=${ncurses}/lib/libpanel${stdenv.hostPlatform.extensions.sharedLibrary}
+      -DLibEdit_INCLUDE_DIRS=${lib.getInclude libedit}/include
+      -DLibEdit_LIBRARIES=${lib.getLib libedit}/lib/libedit${stdenv.hostPlatform.extensions.sharedLibrary}
+      -DCURSES_INCLUDE_DIRS=${lib.getInclude ncurses}/include
+      -DCURSES_LIBRARIES=${lib.getLib ncurses}/lib/libncurses${stdenv.hostPlatform.extensions.sharedLibrary}
+      -DPANEL_LIBRARIES=${lib.getLib ncurses}/lib/libpanel${stdenv.hostPlatform.extensions.sharedLibrary}
     ";
     buildProject lldb llvm-project/lldb
 
@@ -586,7 +582,7 @@ in stdenv.mkDerivation {
       -DSWIFT_DEST_ROOT=$out
       -DSWIFT_HOST_VARIANT_SDK=OSX
 
-      -DSWIFT_DARWIN_DEPLOYMENT_VERSION_OSX=10.15
+      -DSWIFT_DARWIN_DEPLOYMENT_VERSION_OSX=${deploymentVersion}
       -DSWIFT_DARWIN_DEPLOYMENT_VERSION_IOS=13.0
       -DSWIFT_DARWIN_DEPLOYMENT_VERSION_MACCATALYST=13.0
       -DSWIFT_DARWIN_DEPLOYMENT_VERSION_TVOS=13.0
@@ -668,12 +664,6 @@ in stdenv.mkDerivation {
     # just copying the 3 symlinks inside to smaller closures.
     mkdir $lib/lib/swift/clang
     cp -P ${clang}/resource-root/* $lib/lib/swift/clang/
-
-    ${lib.optionalString stdenv.hostPlatform.isDarwin ''
-    # Install required library for ObjC interop.
-    # TODO: Is there no source code for this available?
-    cp -r ${CLTools_Executables}/usr/lib/arc $out/lib/arc
-    ''}
   '';
 
   preFixup = lib.optionalString stdenv.hostPlatform.isLinux ''
@@ -713,7 +703,10 @@ in stdenv.mkDerivation {
     done
 
     wrapProgram $out/bin/swift-frontend \
-      --prefix PATH : ${lib.makeBinPath runtimeDeps}
+      --prefix PATH : ${lib.makeBinPath [ cctools.libtool ]}
+
+    # Needs to be propagated by the compiler not by its dev output.
+    moveToOutput nix-support/propagated-target-target-deps "$out"
   '';
 
   passthru = {

pkgs/development/compilers/swift/default.nix

@@ -5,6 +5,7 @@
 , llvmPackages
 , llvmPackages_15
 , overrideCC
+, overrideLibcxx
 }:
 
 let
@@ -15,7 +16,11 @@ let
     callPackage = newScope self;
 
     # Current versions of Swift on Darwin require macOS SDK 10.15 at least.
-    # Re-export this so we can rely on the minimum Swift SDK elsewhere.
+    # The Swift compiler propagates the 13.3 SDK and a 10.15 deployment target.
+    # Packages that need a newer version can add it to their build inputs
+    # to use it (as normal).
+
+    # This SDK is included for compatibility with existing packages.
     apple_sdk = pkgs.darwin.apple_sdk_11_0;
 
     # Swift builds its own Clang for internal use. We wrap that clang with a
@@ -24,43 +29,24 @@ let
     # we'll often run into compilation errors.
     #
     # The following selects the correct Clang version, matching the version
-    # used in Swift, and applies the same libc overrides as `apple_sdk.stdenv`.
-    clang = if pkgs.stdenv.hostPlatform.isDarwin
-      then
-        swiftLlvmPackages.clang.override rec {
-          libc = apple_sdk.Libsystem;
-          bintools = pkgs.bintools.override { inherit libc; };
-          # Ensure that Swift’s internal clang uses the same libc++ and libc++abi as the
-          # default Darwin stdenv. Using the default libc++ avoids issues (such as crashes)
-          # that can happen when a Swift application dynamically links different versions
-          # of libc++ and libc++abi than libraries it links are using.
-          inherit (llvmPackages) libcxx;
-        }
-      else
-        swiftLlvmPackages.clang;
+    # used in Swift.
+    inherit (swiftLlvmPackages) clang;
 
     # Overrides that create a useful environment for swift packages, allowing
-    # packaging with `swiftPackages.callPackage`. These are similar to
-    # `apple_sdk_11_0.callPackage`, with our clang on top.
+    # packaging with `swiftPackages.callPackage`.
     inherit (clang) bintools;
-    stdenv = overrideCC pkgs.stdenv clang;
-    darwin = pkgs.darwin.overrideScope (_: prev: {
-      inherit apple_sdk;
-      inherit (apple_sdk) Libsystem LibsystemCross libcharset libunwind objc4 configd IOKit Security;
-      CF = apple_sdk.CoreFoundation // { __attrsFailEvaluation = true; };
-      __attrsFailEvaluation = true;
-    });
-    xcodebuild = pkgs.xcbuild.override {
-      inherit (apple_sdk.frameworks) CoreServices CoreGraphics ImageIO;
-      inherit stdenv;
-      sdkVer = "10.15";
-    };
-    xcbuild = xcodebuild;
+    stdenv =
+      let
+        stdenv' = overrideCC pkgs.stdenv clang;
+      in
+      # Ensure that Swift’s internal clang uses the same libc++ and libc++abi as the
+      # default clang’s stdenv. Using the default libc++ avoids issues (such as crashes)
+      # that can happen when a Swift application dynamically links different versions
+      # of libc++ and libc++abi than libraries it links are using.
+      if stdenv'.cc.libcxx != null then overrideLibcxx stdenv' else stdenv';
 
     swift-unwrapped = callPackage ./compiler {
       inherit (darwin) DarwinTools sigtool;
-      inherit (apple_sdk) MacOSX-SDK CLTools_Executables;
-      inherit (apple_sdk.frameworks) CoreServices Foundation Combine;
     };
 
     swiftNoSwiftDriver = callPackage ./wrapper {
@@ -69,11 +55,11 @@ let
     };
 
     Dispatch = if stdenv.hostPlatform.isDarwin
-      then null # part of libsystem
+      then null # part of apple-sdk
       else callPackage ./libdispatch { swift = swiftNoSwiftDriver; };
 
     Foundation = if stdenv.hostPlatform.isDarwin
-      then apple_sdk.frameworks.Foundation
+      then null # part of apple-sdk
       else callPackage ./foundation { swift = swiftNoSwiftDriver; };
 
     # TODO: Apple distributes a binary XCTest with Xcode, but it is not part of

pkgs/development/compilers/swift/swiftpm/default.nix

@@ -17,6 +17,7 @@
 , makeWrapper
 , DarwinTools # sw_vers
 , cctools # vtool
+, darwinMinVersionHook
 , xcbuild
 , CryptoKit
 , LocalAuthentication
@@ -113,13 +114,6 @@ let
         fi
       '';
 
-    preConfigure = (attrs.preConfigure or "")
-      + ''
-        # Builds often don't set a target, and our default minimum macOS deployment
-        # target on x86_64-darwin is too low. Harmless on non-Darwin.
-        export MACOSX_DEPLOYMENT_TARGET=10.15.4
-      '';
-
     postInstall = (attrs.postInstall or "")
       + lib.optionalString stdenv.hostPlatform.isDarwin ''
         # The install name of libraries is incorrectly set to lib/ (via our
@@ -362,7 +356,7 @@ let
       swift-driver
       swift-system
       swift-tools-support-core
-    ];
+    ] ++ lib.optionals stdenv.isDarwin [ (darwinMinVersionHook "10.15.4") ];
 
     cmakeFlags = [
       "-DUSE_CMAKE_INSTALL=ON"
@@ -392,7 +386,7 @@ in stdenv.mkDerivation (commonAttrs // {
     ++ lib.optionals stdenv.hostPlatform.isDarwin [
       CryptoKit
       LocalAuthentication
-    ];
+  ] ++ lib.optionals stdenv.isDarwin [ (darwinMinVersionHook "10.15.4") ];
 
   configurePhase = generated.configure + ''
     # Functionality provided by Xcode XCTest, but not available in
@@ -411,9 +405,6 @@ in stdenv.mkDerivation (commonAttrs // {
   '';
 
   buildPhase = ''
-    # Required to link with swift-corelibs-xctest on Darwin.
-    export SWIFTTSC_MACOS_DEPLOYMENT_TARGET=10.12
-
     TERM=dumb swift-build -c release
   '';
 

pkgs/development/compilers/swift/wrapper/default.nix

@@ -18,6 +18,10 @@ stdenv.mkDerivation (swift._wrapperParams // {
     swiftStaticModuleSubdir swiftStaticLibSubdir;
   swiftDriver = lib.optionalString useSwiftDriver "${swift-driver}/bin/swift-driver";
 
+  env.darwinMinVersion = lib.optionalString stdenv.targetPlatform.isDarwin (
+    stdenv.targetPlatform.darwinMinVersion
+  );
+
   passAsFile = [ "buildCommand" ];
   buildCommand = ''
     mkdir -p $out/bin $out/nix-support
@@ -48,6 +52,13 @@ stdenv.mkDerivation (swift._wrapperParams // {
     ln -s ${swift.lib}/lib $out/lib
 
     substituteAll ${./setup-hook.sh} $out/nix-support/setup-hook
+
+    # Propagate any propagated inputs from the unwrapped Swift compiler, if any.
+    if [ -e "$swift/nix-support" ]; then
+      for input in "$swift/nix-support/"*propagated*; do
+        cp "$input" "$out/nix-support/$(basename "$input")"
+      done
+    fi
   '';
 
   passthru = {

pkgs/development/compilers/swift/wrapper/wrapper.sh

@@ -156,6 +156,14 @@ if [ -z "${NIX_CC_WRAPPER_FLAGS_SET_@suffixSalt@:-}" ]; then
     source $cc_wrapper/nix-support/add-flags.sh
 fi
 
+# Only add darwin min version flag and set up `DEVELOPER_DIR` if a default darwin min version is set,
+# which is a signal that we're targeting darwin. (Copied from add-flags in libc but tailored for Swift).
+if [ "@darwinMinVersion@" ]; then
+    # Make sure the wrapped Swift compiler can find the overlays in the SDK.
+    NIX_SWIFTFLAGS_COMPILE+=" -I $SDKROOT/usr/lib/swift"
+    NIX_LDFLAGS_@suffixSalt@+=" -L $SDKROOT/usr/lib/swift"
+fi
+
 if [[ "$isCxx" = 1 ]]; then
     if [[ "$cxxInclude" = 1 ]]; then
         NIX_CFLAGS_COMPILE_@suffixSalt@+=" $NIX_CXXSTDLIB_COMPILE_@suffixSalt@"