pkgs/top-level: make package sets composable

The various pkgsXYZ top-level package sets did not pass localSystem /
crossSystem to lower levels, so far. This change propagates original
arguments to lower levels, which include the overrides made by an upper
package sets.

There is an extensive test-suite to test various combinations of package
sets in pkgs/test/top-level. There are a few basic promises made:

- Package sets must be idempotent. pkgsMusl.pkgsMusl === pkgsMusl.

- Once pkgsCross is used any subsequent package sets should affect the
  **host platform** and not the build platform. Examples:
  - pkgsMusl.pkgsCross.gnu64 is a cross compilation from musl to glibc
  - pkgsCross.gnu64.pkgsMusl is a cross compilation to musl

- Modifications from an earlier layer should not be lost, unless
  explicitly overwritten. Examples:
  - pkgsStatic.pkgsMusl should still be static.
  - pkgsStatic.pkgsCross.gnu64 should be static, but with glibc instead of
    musl.

Exceptions / TODOs:
- pkgsExtraHardening is currently not idempotent, because it applies the
  same flags over and over again.

Supersedes NixOS#136549
Resolves NixOS#114510
Resolves NixOS#212494
Resolves NixOS#281596

pkgs/test/top-level/default.nix

@@ -105,34 +105,32 @@ lib.recurseIntoAttrs {
       assert isIdempotent "pkgsi686Linux";
       assert isIdempotent "pkgsx86_64Darwin";
 
-      # TODO: fails
-      # assert isNoop [ "pkgsStatic" ] "pkgsMusl";
-      # TODO: fails because of ppc64-musl
-      # assert lib.all (sys: isNoop [ "pkgsCross" sys ] "pkgsMusl") allMuslExamples;
+      assert isNoop [ "pkgsStatic" ] "pkgsMusl";
+      assert lib.all (sys: isNoop [ "pkgsCross" sys ] "pkgsMusl") allMuslExamples;
       assert lib.all (sys: isNoop [ "pkgsCross" sys ] "pkgsLLVM") allLLVMExamples;
 
       assert isComposable "pkgsExtraHardening";
       assert isComposable "pkgsLLVM";
       assert isComposable "pkgsArocc";
       # TODO: unexpected argument 'bintools' - uncomment once https://github.com/NixOS/nixpkgs/pull/331011 is done
       # assert isComposable "pkgsZig";
-      # TODO: attribute 'abi' missing
-      # assert isComposable "pkgsMusl";
-      # TODO: fails
-      # assert isComposable "pkgsStatic";
-      # assert isComposable "pkgsi686Linux";
+      assert isComposable "pkgsMusl";
+      assert isComposable "pkgsStatic";
+      assert isComposable "pkgsi686Linux";
 
       # Special cases regarding buildPlatform vs hostPlatform
-      # TODO: fails
-      # assert discardEvaluationErrors (pkgsCross.gnu64.pkgsMusl.stdenv.hostPlatform.isMusl);
-      # assert discardEvaluationErrors (pkgsCross.gnu64.pkgsi686Linux.stdenv.hostPlatform.isx86_32);
-      # assert discardEvaluationErrors (pkgsCross.mingwW64.pkgsLinux.stdenv.hostPlatform.isLinux);
-      # assert discardEvaluationErrors (pkgsCross.aarch64-darwin.pkgsx86_64Darwin.stdenv.hostPlatform.isx86_64);
+      assert discardEvaluationErrors (pkgsCross.gnu64.pkgsMusl.stdenv.hostPlatform.isMusl);
+      assert discardEvaluationErrors (pkgsCross.gnu64.pkgsi686Linux.stdenv.hostPlatform.isx86_32);
+      assert discardEvaluationErrors (pkgsCross.mingwW64.pkgsLinux.stdenv.hostPlatform.isLinux);
+      assert discardEvaluationErrors (pkgsCross.aarch64-darwin.pkgsx86_64Darwin.stdenv.hostPlatform.isx86_64);
 
       # pkgsCross should keep upper cross settings
-      # TODO: fails
-      # assert discardEvaluationErrors (with pkgsStatic.pkgsCross.gnu64.stdenv.hostPlatform; isGnu && isStatic);
-      # assert discardEvaluationErrors (with pkgsLLVM.pkgsCross.musl64.stdenv.hostPlatform; isMusl && useLLVM);
+      assert discardEvaluationErrors (with pkgsStatic.pkgsCross.gnu64.stdenv.hostPlatform; isGnu && isStatic);
+      assert discardEvaluationErrors (with pkgsLLVM.pkgsCross.musl64.stdenv.hostPlatform; isMusl && useLLVM);
+
+      # pkgsCross should keep upper cross settings
+      assert discardEvaluationErrors (with pkgsStatic.pkgsCross.gnu64.stdenv.hostPlatform; isGnu && isStatic);
+      assert discardEvaluationErrors (with pkgsLLVM.pkgsCross.musl64.stdenv.hostPlatform; isMusl && useLLVM);
 
       emptyFile;
 }

pkgs/top-level/default.nix

@@ -121,7 +121,7 @@ in let
   # via `evalModules` is not idempotent. In other words, if you add `config` to
   # `newArgs`, expect strange very hard to debug errors! (Yes, I'm speaking from
   # experience here.)
-  nixpkgsFun = newArgs: import ./. (args // newArgs);
+  nixpkgsFun = f0: import ./. (args // f0 args);
 
   # Partially apply some arguments for building bootstraping stage pkgs
   # sets. Only apply arguments which no stdenv would want to override.

pkgs/top-level/stage.nix

@@ -178,12 +178,12 @@ let
   # Convenience attributes for instantitating package sets. Each of
   # these will instantiate a new version of allPackages.
   otherPackageSets = let
-    mkPkgs = name: nixpkgsArgs: nixpkgsFun (nixpkgsArgs // {
+    mkPkgs = name: fn: nixpkgsFun (prevArgs: let nixpkgsArgs = fn prevArgs; in nixpkgsArgs // {
       overlays = [
         (self': super': {
           "${name}" = super';
         })
-      ] ++ nixpkgsArgs.overlays or [] ++ overlays;
+      ] ++ nixpkgsArgs.overlays or [] ++ prevArgs.overlays;
     });
   in self: super: {
     # This maps each entry in lib.systems.examples to its own package
@@ -192,94 +192,92 @@ let
     # will refer to the "hello" package built for the ARM6-based
     # Raspberry Pi.
     pkgsCross = lib.mapAttrs (n: crossSystem:
-                              nixpkgsFun { inherit crossSystem; })
+                              nixpkgsFun (prevArgs: { crossSystem = (lib.systems.systemToAttrs (lib.defaultTo { } prevArgs.crossSystem or null)) // crossSystem; }))
                               lib.systems.examples;
 
-    pkgsLLVM = mkPkgs "pkgsLLVM" {
+    pkgsLLVM = mkPkgs "pkgsLLVM" (prevArgs: {
       # Bootstrap a cross stdenv using the LLVM toolchain.
       # This is currently not possible when compiling natively,
-      # so we don't need to check hostPlatform != buildPlatform.
-      crossSystem = stdenv.hostPlatform // {
+      # so we don't need to check whether we are cross already.
+      crossSystem = (lib.systems.systemToAttrs (lib.defaultTo prevArgs.localSystem prevArgs.crossSystem or null)) // {
         useLLVM = true;
         linker = "lld";
       };
-    };
+    });
 
-    pkgsArocc = mkPkgs "pkgsArocc" {
+    pkgsArocc = mkPkgs "pkgsArocc" (prevArgs: {
       # Bootstrap a cross stdenv using the Aro C compiler.
       # This is currently not possible when compiling natively,
-      # so we don't need to check hostPlatform != buildPlatform.
-      crossSystem = stdenv.hostPlatform // {
+      # so we don't need to check whether we are cross already.
+      crossSystem = (lib.systems.systemToAttrs (lib.defaultTo prevArgs.localSystem prevArgs.crossSystem or null)) // {
         useArocc = true;
         linker = "lld";
       };
-    };
+    });
 
-    pkgsZig = mkPkgs "pkgsZig" {
+    pkgsZig = mkPkgs "pkgsZig" (prevArgs: {
       # Bootstrap a cross stdenv using the Zig toolchain.
       # This is currently not possible when compiling natively,
-      # so we don't need to check hostPlatform != buildPlatform.
-      crossSystem = stdenv.hostPlatform // {
+      # so we don't need to check whether we are cross already.
+      crossSystem = (lib.systems.systemToAttrs (lib.defaultTo prevArgs.localSystem prevArgs.crossSystem or null)) // {
         useZig = true;
         linker = "lld";
       };
-    };
+    });
 
     # All packages built with the Musl libc. This will override the
     # default GNU libc on Linux systems. Non-Linux systems are not
     # supported. 32-bit is also not supported.
-    pkgsMusl = if stdenv.hostPlatform.isLinux && stdenv.buildPlatform.is64bit then mkPkgs "pkgsMusl" {
-      ${if stdenv.hostPlatform == stdenv.buildPlatform
-        then "localSystem" else "crossSystem"} = {
+    pkgsMusl = if stdenv.hostPlatform.isLinux && stdenv.buildPlatform.is64bit then mkPkgs "pkgsMusl" (prevArgs: {
+      ${if prevArgs ? crossSystem then "crossSystem" else "localSystem"} = (lib.systems.systemToAttrs (lib.defaultTo prevArgs.localSystem prevArgs.crossSystem or null)) // {
         config = lib.systems.parse.tripleFromSystem (makeMuslParsedPlatform stdenv.hostPlatform.parsed);
       };
-    } else throw "Musl libc only supports 64-bit Linux systems.";
+    }) else throw "Musl libc only supports 64-bit Linux systems.";
 
     # All packages built for i686 Linux.
     # Used by wine, firefox with debugging version of Flash, ...
-    pkgsi686Linux = if stdenv.hostPlatform.isLinux && stdenv.hostPlatform.isx86 then mkPkgs "pkgsi686Linux" {
-      ${if stdenv.hostPlatform == stdenv.buildPlatform
-        then "localSystem" else "crossSystem"} = {
+    pkgsi686Linux = if stdenv.hostPlatform.isLinux && stdenv.hostPlatform.isx86 then mkPkgs "pkgsi686Linux" (prevArgs: {
+      ${if prevArgs ? crossSystem then "crossSystem" else "localSystem"} = (lib.systems.systemToAttrs (lib.defaultTo prevArgs.localSystem prevArgs.crossSystem or null)) // {
         config = lib.systems.parse.tripleFromSystem (stdenv.hostPlatform.parsed // {
           cpu = lib.systems.parse.cpuTypes.i686;
         });
       };
-    } else throw "i686 Linux package set can only be used with the x86 family.";
+    }) else throw "i686 Linux package set can only be used with the x86 family.";
 
     # x86_64-darwin packages for aarch64-darwin users to use with Rosetta for incompatible packages
-    pkgsx86_64Darwin = if stdenv.hostPlatform.isDarwin then mkPkgs "pkgsx86_64Darwin" {
-      localSystem = {
+    pkgsx86_64Darwin = if stdenv.hostPlatform.isDarwin then mkPkgs "pkgsx86_64Darwin" (prevArgs: {
+      ${if prevArgs ? crossSystem then "crossSystem" else "localSystem"} = (lib.systems.systemToAttrs (lib.defaultTo prevArgs.localSystem prevArgs.crossSystem or null)) // {
         config = lib.systems.parse.tripleFromSystem (stdenv.hostPlatform.parsed // {
           cpu = lib.systems.parse.cpuTypes.x86_64;
         });
       };
-    } else throw "x86_64 Darwin package set can only be used on Darwin systems.";
+    }) else throw "x86_64 Darwin package set can only be used on Darwin systems.";
 
     # If already linux: the same package set unaltered
-    # Otherwise, return a natively built linux package set for the current cpu architecture string.
+    # Otherwise, return a linux package set for the current cpu architecture string.
+    # (ABI and other details will be set to the default for the cpu/os pair)
     pkgsLinux =
       if stdenv.hostPlatform.isLinux
       then self
-      else mkPkgs "pkgsLinux" {
-        localSystem = lib.systems.elaborate "${stdenv.hostPlatform.parsed.cpu.name}-linux";
-      };
+      else mkPkgs "pkgsLinux" (prevArgs: {
+        ${if prevArgs ? crossSystem then "crossSystem" else "localSystem"} = (lib.systems.systemToAttrs (lib.defaultTo prevArgs.localSystem prevArgs.crossSystem or null)) // {
+          config = lib.systems.parse.tripleFromSystem (lib.systems.elaborate "${stdenv.hostPlatform.parsed.cpu.name}-linux").parsed;
+        };
+      });
 
     # Fully static packages.
     # Currently uses Musl on Linux (couldn’t get static glibc to work).
-    pkgsStatic = mkPkgs "pkgsStatic" {
-      crossSystem = {
+    pkgsStatic = mkPkgs "pkgsStatic" (prevArgs: {
+      crossSystem = (lib.systems.systemToAttrs (lib.defaultTo prevArgs.localSystem prevArgs.crossSystem or null)) // {
         isStatic = true;
-        config = lib.systems.parse.tripleFromSystem (
-          if stdenv.hostPlatform.isLinux
-          then makeMuslParsedPlatform stdenv.hostPlatform.parsed
-          else stdenv.hostPlatform.parsed
-        );
-        gcc = lib.optionalAttrs (stdenv.hostPlatform.system == "powerpc64-linux") { abi = "elfv2"; } //
-          stdenv.hostPlatform.gcc or {};
+      } // lib.optionalAttrs stdenv.hostPlatform.isLinux {
+        config = lib.systems.parse.tripleFromSystem (makeMuslParsedPlatform stdenv.hostPlatform.parsed);
+      } // lib.optionalAttrs (stdenv.hostPlatform.system == "powerpc64-linux") {
+        gcc = { abi = "elfv2"; } // stdenv.hostPlatform.gcc or {};
       };
-    };
+    });
 
-    pkgsExtraHardening = mkPkgs "pkgsExtraHardening" {
+    pkgsExtraHardening = mkPkgs "pkgsExtraHardening" (_: {
       overlays = [
         (self': super': {
           stdenv = super'.withDefaultHardeningFlags (
@@ -301,15 +299,15 @@ let
           pcre16 = super'.pcre16.override { enableJit = false; };
         })
       ];
-    };
+    });
 
     # Extend the package set with zero or more overlays. This preserves
     # preexisting overlays. Prefer to initialize with the right overlays
     # in one go when calling Nixpkgs, for performance and simplicity.
     appendOverlays = extraOverlays:
       if extraOverlays == []
       then self
-      else nixpkgsFun { overlays = args.overlays ++ extraOverlays; };
+      else nixpkgsFun (prevArgs: { overlays = prevArgs.overlays ++ extraOverlays; });
 
     # NOTE: each call to extend causes a full nixpkgs rebuild, adding ~130MB
     #       of allocations. DO NOT USE THIS IN NIXPKGS.