Skip to content

Commit

Permalink
pkgs/top-level: make package sets composable
Browse files Browse the repository at this point in the history 
The various pkgsXYZ top-level package sets did not pass localSystem /
crossSystem to lower levels, so far. This change propagates original
arguments to lower levels, which include the overrides made by an upper
package sets.

There is an extensive test-suite to test various combinations of package
sets in pkgs/test/top-level. There are a few basic promises made:

- Package sets must be idempotent. pkgsMusl.pkgsMusl === pkgsMusl.

- Once pkgsCross is used any subsequent package sets should affect the
  **host platform** and not the build platform. Examples:
  - pkgsMusl.pkgsCross.aarch64-multiplatform is a cross compilation from
musl to glibc/aarch64
  - pkgsCross.aarch64-multiplatform.pkgsMusl is a cross compilation to
musl/aarch64

- Modifications from an earlier layer should not be lost, unless
  explicitly overwritten. Examples:
  - pkgsStatic.pkgsMusl should still be static.
  - pkgsStatic.pkgsCross.gnu64 should be static, but with glibc instead
of musl.

Exceptions / TODOs:
- pkgsExtraHardening is currently not idempotent, because it applies the
  same flags over and over again.

Supersedes NixOS#136549
Resolves NixOS#114510
Resolves NixOS#212494
Resolves NixOS#281596
  • Loading branch information
@wolfgangwalther
wolfgangwalther committed Jan 25, 2025
1 parent 8cb7998 commit 5124948
Show file tree
Hide file tree
Showing 3 changed files with 67 additions and 64 deletions.
37 changes: 19 additions & 18 deletions pkgs/test/top-level/stage.nix
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,35 +91,36 @@ assert isIdempotent "pkgsStatic";
assert isIdempotent "pkgsi686Linux";
assert isIdempotent "pkgsx86_64Darwin";

# TODO: fails
# assert isNoop [ "pkgsStatic" ] "pkgsMusl";
# TODO: fails because of ppc64-musl
# assert lib.all (sys: isNoop [ "pkgsCross" sys ] "pkgsMusl") allMuslExamples;
assert isNoop [ "pkgsStatic" ] "pkgsMusl";
assert lib.all (sys: isNoop [ "pkgsCross" sys ] "pkgsMusl") allMuslExamples;
assert lib.all (sys: isNoop [ "pkgsCross" sys ] "pkgsLLVM") allLLVMExamples;

# TODO: fails
# assert isComposable "pkgsExtraHardening";
# assert isComposable "pkgsLLVM";
assert isComposable "pkgsExtraHardening";
assert isComposable "pkgsLLVM";
# TODO: Results in infinite recursion
# assert isComposable "pkgsLLVMLibc";
# assert isComposable "pkgsArocc";
assert isComposable "pkgsArocc";
# TODO: unexpected argument 'bintools' - uncomment once https://github.com/NixOS/nixpkgs/pull/331011 is done
# assert isComposable "pkgsZig";
# TODO: attribute 'abi' missing
# assert isComposable "pkgsMusl";
# TODO: fails
# assert isComposable "pkgsStatic";
# assert isComposable "pkgsi686Linux";
assert isComposable "pkgsMusl";
assert isComposable "pkgsStatic";
assert isComposable "pkgsi686Linux";

# Special cases regarding buildPlatform vs hostPlatform
# TODO: fails
# assert discardEvaluationErrors (pkgsCross.gnu64.pkgsMusl.stdenv.hostPlatform.isMusl);
# assert discardEvaluationErrors (pkgsCross.gnu64.pkgsi686Linux.stdenv.hostPlatform.isx86_32);
# assert discardEvaluationErrors (pkgsCross.mingwW64.pkgsLinux.stdenv.hostPlatform.isLinux);
# assert discardEvaluationErrors (pkgsCross.aarch64-darwin.pkgsx86_64Darwin.stdenv.hostPlatform.isx86_64);
assert discardEvaluationErrors (pkgsCross.mingwW64.pkgsLinux.stdenv.hostPlatform.isLinux);
assert discardEvaluationErrors (
pkgsCross.aarch64-darwin.pkgsx86_64Darwin.stdenv.hostPlatform.isx86_64
);

# pkgsCross should keep upper cross settings
# TODO: fails
# assert discardEvaluationErrors (with pkgsStatic.pkgsCross.gnu64.stdenv.hostPlatform; isGnu && isStatic);
# assert discardEvaluationErrors (with pkgsLLVM.pkgsCross.musl64.stdenv.hostPlatform; isMusl && useLLVM);
assert discardEvaluationErrors (
with pkgsStatic.pkgsCross.gnu64.stdenv.hostPlatform; isGnu && isStatic
);
assert discardEvaluationErrors (
with pkgsLLVM.pkgsCross.musl64.stdenv.hostPlatform; isMusl && useLLVM
);

emptyFile
2 changes: 1 addition & 1 deletion pkgs/top-level/default.nix
View file
Original file line number Diff line number Diff line change
Expand Up @@ -142,7 +142,7 @@ in let
# via `evalModules` is not idempotent. In other words, if you add `config` to
# `newArgs`, expect strange very hard to debug errors! (Yes, I'm speaking from
# experience here.)
nixpkgsFun = newArgs: import ./. (args // newArgs);
nixpkgsFun = f0: import ./. (args // f0 args);

# Partially apply some arguments for building bootstraping stage pkgs
# sets. Only apply arguments which no stdenv would want to override.
Expand Down
92 changes: 47 additions & 45 deletions pkgs/top-level/stage.nix
View file
Original file line number Diff line number Diff line change
Expand Up @@ -182,12 +182,12 @@ let
# Convenience attributes for instantitating package sets. Each of
# these will instantiate a new version of allPackages.
otherPackageSets = let
mkPkgs = name: nixpkgsArgs: nixpkgsFun (nixpkgsArgs // {
mkPkgs = name: fn: nixpkgsFun (prevArgs: let nixpkgsArgs = fn prevArgs; in nixpkgsArgs // {
overlays = [
(self': super': {
"${name}" = super';
})
] ++ nixpkgsArgs.overlays or [] ++ overlays;
] ++ nixpkgsArgs.overlays or [] ++ prevArgs.overlays or [];
});
in self: super: {
# This maps each entry in lib.systems.examples to its own package
Expand All @@ -196,104 +196,106 @@ let
# will refer to the "hello" package built for the ARM6-based
# Raspberry Pi.
pkgsCross = lib.mapAttrs (n: crossSystem:
nixpkgsFun { inherit crossSystem; })
nixpkgsFun (prevArgs: { crossSystem = (lib.systems.systemToAttrs (lib.defaultTo { } prevArgs.crossSystem or null)) // crossSystem; }))
lib.systems.examples;

pkgsLLVM = mkPkgs "pkgsLLVM" {
pkgsLLVM = mkPkgs "pkgsLLVM" (prevArgs: {
# Bootstrap a cross stdenv using the LLVM toolchain.
# This is currently not possible when compiling natively,
# so we don't need to check hostPlatform != buildPlatform.
crossSystem = stdenv.hostPlatform // {
# so we don't need to check whether we are cross already.
crossSystem = (lib.systems.systemToAttrs (lib.defaultTo prevArgs.localSystem prevArgs.crossSystem or null)) // {
useLLVM = true;
linker = "lld";
};
};
});

pkgsLLVMLibc = mkPkgs "pkgsLLVMLibc" {
pkgsLLVMLibc = mkPkgs "pkgsLLVMLibc" (prevArgs: {
# Bootstrap a cross stdenv using LLVM libc.
# This is currently not possible when compiling natively,
# so we don't need to check hostPlatform != buildPlatform.
crossSystem = stdenv.hostPlatform // {
# so we don't need to check whether we are cross already.
crossSystem = (lib.systems.systemToAttrs (lib.defaultTo prevArgs.localSystem prevArgs.crossSystem or null)) // {
config = lib.systems.parse.tripleFromSystem (makeLLVMParsedPlatform stdenv.hostPlatform.parsed);
libc = "llvm";
};
};
});

pkgsArocc = mkPkgs "pkgsArocc" {
pkgsArocc = mkPkgs "pkgsArocc" (prevArgs: {
# Bootstrap a cross stdenv using the Aro C compiler.
# This is currently not possible when compiling natively,
# so we don't need to check hostPlatform != buildPlatform.
crossSystem = stdenv.hostPlatform // {
# so we don't need to check whether we are cross already.
crossSystem = (lib.systems.systemToAttrs (lib.defaultTo prevArgs.localSystem prevArgs.crossSystem or null)) // {
useArocc = true;
linker = "lld";
};
};
});

pkgsZig = mkPkgs "pkgsZig" {
pkgsZig = mkPkgs "pkgsZig" (prevArgs: {
# Bootstrap a cross stdenv using the Zig toolchain.
# This is currently not possible when compiling natively,
# so we don't need to check hostPlatform != buildPlatform.
crossSystem = stdenv.hostPlatform // {
# so we don't need to check whether we are cross already.
crossSystem = (lib.systems.systemToAttrs (lib.defaultTo prevArgs.localSystem prevArgs.crossSystem or null)) // {
useZig = true;
linker = "lld";
};
};
});

# All packages built with the Musl libc. This will override the
# default GNU libc on Linux systems. Non-Linux systems are not
# supported. 32-bit is also not supported.
pkgsMusl = if stdenv.hostPlatform.isLinux && stdenv.buildPlatform.is64bit then mkPkgs "pkgsMusl" {
pkgsMusl = if stdenv.hostPlatform.isLinux && stdenv.buildPlatform.is64bit then mkPkgs "pkgsMusl" (prevArgs: {
${if stdenv.hostPlatform == stdenv.buildPlatform
then "localSystem" else "crossSystem"} = {
then "localSystem" else "crossSystem"} = (lib.systems.systemToAttrs (lib.defaultTo prevArgs.localSystem prevArgs.crossSystem or null)) // {
config = lib.systems.parse.tripleFromSystem (makeMuslParsedPlatform stdenv.hostPlatform.parsed);
};
} else throw "Musl libc only supports 64-bit Linux systems.";
}) else throw "Musl libc only supports 64-bit Linux systems.";

# All packages built for i686 Linux.
# Used by wine, firefox with debugging version of Flash, ...
pkgsi686Linux = if stdenv.hostPlatform.isLinux && stdenv.hostPlatform.isx86 then mkPkgs "pkgsi686Linux" {
pkgsi686Linux = if stdenv.hostPlatform.isLinux && stdenv.hostPlatform.isx86 then mkPkgs "pkgsi686Linux" (prevArgs: {
${if stdenv.hostPlatform == stdenv.buildPlatform
then "localSystem" else "crossSystem"} = {
then "localSystem" else "crossSystem"} = (lib.systems.systemToAttrs (lib.defaultTo prevArgs.localSystem prevArgs.crossSystem or null)) // {
config = lib.systems.parse.tripleFromSystem (stdenv.hostPlatform.parsed // {
cpu = lib.systems.parse.cpuTypes.i686;
});
};
} else throw "i686 Linux package set can only be used with the x86 family.";
}) else throw "i686 Linux package set can only be used with the x86 family.";

# x86_64-darwin packages for aarch64-darwin users to use with Rosetta for incompatible packages
pkgsx86_64Darwin = if stdenv.hostPlatform.isDarwin then mkPkgs "pkgsx86_64Darwin" {
localSystem = {
pkgsx86_64Darwin = if stdenv.hostPlatform.isDarwin then mkPkgs "pkgsx86_64Darwin" (prevArgs: {
${if stdenv.hostPlatform == stdenv.buildPlatform
then "localSystem" else "crossSystem"} = (lib.systems.systemToAttrs (lib.defaultTo prevArgs.localSystem prevArgs.crossSystem or null)) // {
config = lib.systems.parse.tripleFromSystem (stdenv.hostPlatform.parsed // {
cpu = lib.systems.parse.cpuTypes.x86_64;
});
};
} else throw "x86_64 Darwin package set can only be used on Darwin systems.";
}) else throw "x86_64 Darwin package set can only be used on Darwin systems.";

# If already linux: the same package set unaltered
# Otherwise, return a natively built linux package set for the current cpu architecture string.
# Otherwise, return a linux package set for the current cpu architecture string.
# (ABI and other details will be set to the default for the cpu/os pair)
pkgsLinux =
if stdenv.hostPlatform.isLinux
then self
else mkPkgs "pkgsLinux" {
localSystem = lib.systems.elaborate "${stdenv.hostPlatform.parsed.cpu.name}-linux";
};
else mkPkgs "pkgsLinux" (prevArgs: {
${if stdenv.hostPlatform == stdenv.buildPlatform
then "localSystem" else "crossSystem"} = (lib.systems.systemToAttrs (lib.defaultTo prevArgs.localSystem prevArgs.crossSystem or null)) // {
config = lib.systems.parse.tripleFromSystem (lib.systems.elaborate "${stdenv.hostPlatform.parsed.cpu.name}-linux").parsed;
};
});

# Fully static packages.
# Currently uses Musl on Linux (couldn’t get static glibc to work).
pkgsStatic = mkPkgs "pkgsStatic" {
crossSystem = {
pkgsStatic = mkPkgs "pkgsStatic" (prevArgs: {
crossSystem = (lib.systems.systemToAttrs (lib.defaultTo prevArgs.localSystem prevArgs.crossSystem or null)) // {
isStatic = true;
config = lib.systems.parse.tripleFromSystem (
if stdenv.hostPlatform.isLinux
then makeMuslParsedPlatform stdenv.hostPlatform.parsed
else stdenv.hostPlatform.parsed
);
gcc = lib.optionalAttrs (stdenv.hostPlatform.system == "powerpc64-linux") { abi = "elfv2"; } //
stdenv.hostPlatform.gcc or {};
} // lib.optionalAttrs stdenv.hostPlatform.isLinux {
config = lib.systems.parse.tripleFromSystem (makeMuslParsedPlatform stdenv.hostPlatform.parsed);
} // lib.optionalAttrs (stdenv.hostPlatform.system == "powerpc64-linux") {
gcc = { abi = "elfv2"; } // stdenv.hostPlatform.gcc or {};
};
};
});

pkgsExtraHardening = mkPkgs "pkgsExtraHardening" {
pkgsExtraHardening = mkPkgs "pkgsExtraHardening" (_: {
overlays = [
(self': super': {
stdenv = super'.withDefaultHardeningFlags (
Expand All @@ -315,15 +317,15 @@ let
pcre16 = super'.pcre16.override { enableJit = false; };
})
];
};
});

# Extend the package set with zero or more overlays. This preserves
# preexisting overlays. Prefer to initialize with the right overlays
# in one go when calling Nixpkgs, for performance and simplicity.
appendOverlays = extraOverlays:
if extraOverlays == []
then self
else nixpkgsFun { overlays = args.overlays ++ extraOverlays; };
else nixpkgsFun (prevArgs: { overlays = prevArgs.overlays ++ extraOverlays; });

# NOTE: each call to extend causes a full nixpkgs rebuild, adding ~130MB
# of allocations. DO NOT USE THIS IN NIXPKGS.
Expand Down

0 comments on commit 5124948

Please sign in to comment.