From 5124948f50cfb189e9c33377ae84f824f2e17583 Mon Sep 17 00:00:00 2001 From: Wolfgang Walther Date: Tue, 9 Apr 2024 08:41:18 +0200 Subject: [PATCH] pkgs/top-level: make package sets composable The various pkgsXYZ top-level package sets did not pass localSystem / crossSystem to lower levels, so far. This change propagates original arguments to lower levels, which include the overrides made by an upper package sets. There is an extensive test-suite to test various combinations of package sets in pkgs/test/top-level. There are a few basic promises made: - Package sets must be idempotent. pkgsMusl.pkgsMusl === pkgsMusl. - Once pkgsCross is used any subsequent package sets should affect the **host platform** and not the build platform. Examples: - pkgsMusl.pkgsCross.aarch64-multiplatform is a cross compilation from musl to glibc/aarch64 - pkgsCross.aarch64-multiplatform.pkgsMusl is a cross compilation to musl/aarch64 - Modifications from an earlier layer should not be lost, unless explicitly overwritten. Examples: - pkgsStatic.pkgsMusl should still be static. - pkgsStatic.pkgsCross.gnu64 should be static, but with glibc instead of musl. Exceptions / TODOs: - pkgsExtraHardening is currently not idempotent, because it applies the same flags over and over again. Supersedes #136549 Resolves #114510 Resolves #212494 Resolves #281596 --- pkgs/test/top-level/stage.nix | 37 +++++++------- pkgs/top-level/default.nix | 2 +- pkgs/top-level/stage.nix | 92 ++++++++++++++++++----------------- 3 files changed, 67 insertions(+), 64 deletions(-) diff --git a/pkgs/test/top-level/stage.nix b/pkgs/test/top-level/stage.nix index c207d2ebc374dd..20b28f10ddad50 100644 --- a/pkgs/test/top-level/stage.nix +++ b/pkgs/test/top-level/stage.nix @@ -91,35 +91,36 @@ assert isIdempotent "pkgsStatic"; assert isIdempotent "pkgsi686Linux"; assert isIdempotent "pkgsx86_64Darwin"; -# TODO: fails -# assert isNoop [ "pkgsStatic" ] "pkgsMusl"; -# TODO: fails because of ppc64-musl -# assert lib.all (sys: isNoop [ "pkgsCross" sys ] "pkgsMusl") allMuslExamples; +assert isNoop [ "pkgsStatic" ] "pkgsMusl"; +assert lib.all (sys: isNoop [ "pkgsCross" sys ] "pkgsMusl") allMuslExamples; assert lib.all (sys: isNoop [ "pkgsCross" sys ] "pkgsLLVM") allLLVMExamples; -# TODO: fails -# assert isComposable "pkgsExtraHardening"; -# assert isComposable "pkgsLLVM"; +assert isComposable "pkgsExtraHardening"; +assert isComposable "pkgsLLVM"; +# TODO: Results in infinite recursion # assert isComposable "pkgsLLVMLibc"; -# assert isComposable "pkgsArocc"; +assert isComposable "pkgsArocc"; # TODO: unexpected argument 'bintools' - uncomment once https://github.com/NixOS/nixpkgs/pull/331011 is done # assert isComposable "pkgsZig"; -# TODO: attribute 'abi' missing -# assert isComposable "pkgsMusl"; -# TODO: fails -# assert isComposable "pkgsStatic"; -# assert isComposable "pkgsi686Linux"; +assert isComposable "pkgsMusl"; +assert isComposable "pkgsStatic"; +assert isComposable "pkgsi686Linux"; # Special cases regarding buildPlatform vs hostPlatform # TODO: fails # assert discardEvaluationErrors (pkgsCross.gnu64.pkgsMusl.stdenv.hostPlatform.isMusl); # assert discardEvaluationErrors (pkgsCross.gnu64.pkgsi686Linux.stdenv.hostPlatform.isx86_32); -# assert discardEvaluationErrors (pkgsCross.mingwW64.pkgsLinux.stdenv.hostPlatform.isLinux); -# assert discardEvaluationErrors (pkgsCross.aarch64-darwin.pkgsx86_64Darwin.stdenv.hostPlatform.isx86_64); +assert discardEvaluationErrors (pkgsCross.mingwW64.pkgsLinux.stdenv.hostPlatform.isLinux); +assert discardEvaluationErrors ( + pkgsCross.aarch64-darwin.pkgsx86_64Darwin.stdenv.hostPlatform.isx86_64 +); # pkgsCross should keep upper cross settings -# TODO: fails -# assert discardEvaluationErrors (with pkgsStatic.pkgsCross.gnu64.stdenv.hostPlatform; isGnu && isStatic); -# assert discardEvaluationErrors (with pkgsLLVM.pkgsCross.musl64.stdenv.hostPlatform; isMusl && useLLVM); +assert discardEvaluationErrors ( + with pkgsStatic.pkgsCross.gnu64.stdenv.hostPlatform; isGnu && isStatic +); +assert discardEvaluationErrors ( + with pkgsLLVM.pkgsCross.musl64.stdenv.hostPlatform; isMusl && useLLVM +); emptyFile diff --git a/pkgs/top-level/default.nix b/pkgs/top-level/default.nix index 918a47639f3c68..6787793dbdfc70 100644 --- a/pkgs/top-level/default.nix +++ b/pkgs/top-level/default.nix @@ -142,7 +142,7 @@ in let # via `evalModules` is not idempotent. In other words, if you add `config` to # `newArgs`, expect strange very hard to debug errors! (Yes, I'm speaking from # experience here.) - nixpkgsFun = newArgs: import ./. (args // newArgs); + nixpkgsFun = f0: import ./. (args // f0 args); # Partially apply some arguments for building bootstraping stage pkgs # sets. Only apply arguments which no stdenv would want to override. diff --git a/pkgs/top-level/stage.nix b/pkgs/top-level/stage.nix index 781dcd357cd49e..7884e2378f3646 100644 --- a/pkgs/top-level/stage.nix +++ b/pkgs/top-level/stage.nix @@ -182,12 +182,12 @@ let # Convenience attributes for instantitating package sets. Each of # these will instantiate a new version of allPackages. otherPackageSets = let - mkPkgs = name: nixpkgsArgs: nixpkgsFun (nixpkgsArgs // { + mkPkgs = name: fn: nixpkgsFun (prevArgs: let nixpkgsArgs = fn prevArgs; in nixpkgsArgs // { overlays = [ (self': super': { "${name}" = super'; }) - ] ++ nixpkgsArgs.overlays or [] ++ overlays; + ] ++ nixpkgsArgs.overlays or [] ++ prevArgs.overlays or []; }); in self: super: { # This maps each entry in lib.systems.examples to its own package @@ -196,104 +196,106 @@ let # will refer to the "hello" package built for the ARM6-based # Raspberry Pi. pkgsCross = lib.mapAttrs (n: crossSystem: - nixpkgsFun { inherit crossSystem; }) + nixpkgsFun (prevArgs: { crossSystem = (lib.systems.systemToAttrs (lib.defaultTo { } prevArgs.crossSystem or null)) // crossSystem; })) lib.systems.examples; - pkgsLLVM = mkPkgs "pkgsLLVM" { + pkgsLLVM = mkPkgs "pkgsLLVM" (prevArgs: { # Bootstrap a cross stdenv using the LLVM toolchain. # This is currently not possible when compiling natively, - # so we don't need to check hostPlatform != buildPlatform. - crossSystem = stdenv.hostPlatform // { + # so we don't need to check whether we are cross already. + crossSystem = (lib.systems.systemToAttrs (lib.defaultTo prevArgs.localSystem prevArgs.crossSystem or null)) // { useLLVM = true; linker = "lld"; }; - }; + }); - pkgsLLVMLibc = mkPkgs "pkgsLLVMLibc" { + pkgsLLVMLibc = mkPkgs "pkgsLLVMLibc" (prevArgs: { # Bootstrap a cross stdenv using LLVM libc. # This is currently not possible when compiling natively, - # so we don't need to check hostPlatform != buildPlatform. - crossSystem = stdenv.hostPlatform // { + # so we don't need to check whether we are cross already. + crossSystem = (lib.systems.systemToAttrs (lib.defaultTo prevArgs.localSystem prevArgs.crossSystem or null)) // { config = lib.systems.parse.tripleFromSystem (makeLLVMParsedPlatform stdenv.hostPlatform.parsed); libc = "llvm"; }; - }; + }); - pkgsArocc = mkPkgs "pkgsArocc" { + pkgsArocc = mkPkgs "pkgsArocc" (prevArgs: { # Bootstrap a cross stdenv using the Aro C compiler. # This is currently not possible when compiling natively, - # so we don't need to check hostPlatform != buildPlatform. - crossSystem = stdenv.hostPlatform // { + # so we don't need to check whether we are cross already. + crossSystem = (lib.systems.systemToAttrs (lib.defaultTo prevArgs.localSystem prevArgs.crossSystem or null)) // { useArocc = true; linker = "lld"; }; - }; + }); - pkgsZig = mkPkgs "pkgsZig" { + pkgsZig = mkPkgs "pkgsZig" (prevArgs: { # Bootstrap a cross stdenv using the Zig toolchain. # This is currently not possible when compiling natively, - # so we don't need to check hostPlatform != buildPlatform. - crossSystem = stdenv.hostPlatform // { + # so we don't need to check whether we are cross already. + crossSystem = (lib.systems.systemToAttrs (lib.defaultTo prevArgs.localSystem prevArgs.crossSystem or null)) // { useZig = true; linker = "lld"; }; - }; + }); # All packages built with the Musl libc. This will override the # default GNU libc on Linux systems. Non-Linux systems are not # supported. 32-bit is also not supported. - pkgsMusl = if stdenv.hostPlatform.isLinux && stdenv.buildPlatform.is64bit then mkPkgs "pkgsMusl" { + pkgsMusl = if stdenv.hostPlatform.isLinux && stdenv.buildPlatform.is64bit then mkPkgs "pkgsMusl" (prevArgs: { ${if stdenv.hostPlatform == stdenv.buildPlatform - then "localSystem" else "crossSystem"} = { + then "localSystem" else "crossSystem"} = (lib.systems.systemToAttrs (lib.defaultTo prevArgs.localSystem prevArgs.crossSystem or null)) // { config = lib.systems.parse.tripleFromSystem (makeMuslParsedPlatform stdenv.hostPlatform.parsed); }; - } else throw "Musl libc only supports 64-bit Linux systems."; + }) else throw "Musl libc only supports 64-bit Linux systems."; # All packages built for i686 Linux. # Used by wine, firefox with debugging version of Flash, ... - pkgsi686Linux = if stdenv.hostPlatform.isLinux && stdenv.hostPlatform.isx86 then mkPkgs "pkgsi686Linux" { + pkgsi686Linux = if stdenv.hostPlatform.isLinux && stdenv.hostPlatform.isx86 then mkPkgs "pkgsi686Linux" (prevArgs: { ${if stdenv.hostPlatform == stdenv.buildPlatform - then "localSystem" else "crossSystem"} = { + then "localSystem" else "crossSystem"} = (lib.systems.systemToAttrs (lib.defaultTo prevArgs.localSystem prevArgs.crossSystem or null)) // { config = lib.systems.parse.tripleFromSystem (stdenv.hostPlatform.parsed // { cpu = lib.systems.parse.cpuTypes.i686; }); }; - } else throw "i686 Linux package set can only be used with the x86 family."; + }) else throw "i686 Linux package set can only be used with the x86 family."; # x86_64-darwin packages for aarch64-darwin users to use with Rosetta for incompatible packages - pkgsx86_64Darwin = if stdenv.hostPlatform.isDarwin then mkPkgs "pkgsx86_64Darwin" { - localSystem = { + pkgsx86_64Darwin = if stdenv.hostPlatform.isDarwin then mkPkgs "pkgsx86_64Darwin" (prevArgs: { + ${if stdenv.hostPlatform == stdenv.buildPlatform + then "localSystem" else "crossSystem"} = (lib.systems.systemToAttrs (lib.defaultTo prevArgs.localSystem prevArgs.crossSystem or null)) // { config = lib.systems.parse.tripleFromSystem (stdenv.hostPlatform.parsed // { cpu = lib.systems.parse.cpuTypes.x86_64; }); }; - } else throw "x86_64 Darwin package set can only be used on Darwin systems."; + }) else throw "x86_64 Darwin package set can only be used on Darwin systems."; # If already linux: the same package set unaltered - # Otherwise, return a natively built linux package set for the current cpu architecture string. + # Otherwise, return a linux package set for the current cpu architecture string. + # (ABI and other details will be set to the default for the cpu/os pair) pkgsLinux = if stdenv.hostPlatform.isLinux then self - else mkPkgs "pkgsLinux" { - localSystem = lib.systems.elaborate "${stdenv.hostPlatform.parsed.cpu.name}-linux"; - }; + else mkPkgs "pkgsLinux" (prevArgs: { + ${if stdenv.hostPlatform == stdenv.buildPlatform + then "localSystem" else "crossSystem"} = (lib.systems.systemToAttrs (lib.defaultTo prevArgs.localSystem prevArgs.crossSystem or null)) // { + config = lib.systems.parse.tripleFromSystem (lib.systems.elaborate "${stdenv.hostPlatform.parsed.cpu.name}-linux").parsed; + }; + }); # Fully static packages. # Currently uses Musl on Linux (couldn’t get static glibc to work). - pkgsStatic = mkPkgs "pkgsStatic" { - crossSystem = { + pkgsStatic = mkPkgs "pkgsStatic" (prevArgs: { + crossSystem = (lib.systems.systemToAttrs (lib.defaultTo prevArgs.localSystem prevArgs.crossSystem or null)) // { isStatic = true; - config = lib.systems.parse.tripleFromSystem ( - if stdenv.hostPlatform.isLinux - then makeMuslParsedPlatform stdenv.hostPlatform.parsed - else stdenv.hostPlatform.parsed - ); - gcc = lib.optionalAttrs (stdenv.hostPlatform.system == "powerpc64-linux") { abi = "elfv2"; } // - stdenv.hostPlatform.gcc or {}; + } // lib.optionalAttrs stdenv.hostPlatform.isLinux { + config = lib.systems.parse.tripleFromSystem (makeMuslParsedPlatform stdenv.hostPlatform.parsed); + } // lib.optionalAttrs (stdenv.hostPlatform.system == "powerpc64-linux") { + gcc = { abi = "elfv2"; } // stdenv.hostPlatform.gcc or {}; }; - }; + }); - pkgsExtraHardening = mkPkgs "pkgsExtraHardening" { + pkgsExtraHardening = mkPkgs "pkgsExtraHardening" (_: { overlays = [ (self': super': { stdenv = super'.withDefaultHardeningFlags ( @@ -315,7 +317,7 @@ let pcre16 = super'.pcre16.override { enableJit = false; }; }) ]; - }; + }); # Extend the package set with zero or more overlays. This preserves # preexisting overlays. Prefer to initialize with the right overlays @@ -323,7 +325,7 @@ let appendOverlays = extraOverlays: if extraOverlays == [] then self - else nixpkgsFun { overlays = args.overlays ++ extraOverlays; }; + else nixpkgsFun (prevArgs: { overlays = prevArgs.overlays ++ extraOverlays; }); # NOTE: each call to extend causes a full nixpkgs rebuild, adding ~130MB # of allocations. DO NOT USE THIS IN NIXPKGS.