diff --git a/README.md b/README.md index 00ed7e3..5e55de2 100644 --- a/README.md +++ b/README.md @@ -4,3 +4,55 @@ [![Build Status](https://travis-ci.com/kolloch/nur-packages.svg?branch=master)](https://travis-ci.com/kolloch/nur-packages) [![Cachix Cache](https://img.shields.io/badge/cachix-eigenvalue-blue.svg)](https://eigenvalue.cachix.org) + +## rerunFixedDerivationOnChange + +Creates a fixed output derivation that reruns if any of its inputs change. + +Type: `rerunFixedDerivationOnChange -> derivation -> derivation` + +Example: + +```nix +let myFixedOutputDerivation = pkgs.stdenv.mkDerivation { + # ... + outputHash = "sha256:..."; + outputHashMode = "recursive"; +}; +in +nur.repos.kolloch.lib.rerunFixedDerivationOnChange myFixedOutputDerivation; +``` + +Usually, fixed output derivations are only rerun if their name or hash +changes (i.e. when their output path changes). This makes the derivation +definition irrelevant if the output is in the cache. + +With this helper, you can make sure that the commands that are specified +where at least ran once -- with the exact versions of the buildInputs that +you specified. It does so by putting a hash of all the inputs into the +name. + +Caveat: It uses "import from derivation" under the hood. + +### Implementation + +Getting the hash is easy, in principle, because nix already does that work +for us when creating the output paths for non-fixed output derivations. +Therefore, if we create a non-fixed output derivation with the same inputs, +the hash in the output path should change with every change in inputs. +Unfortunately, we cannot use parts of the output path in the name of our +derivation directly. + +Nix prevents this because supposedly it is not what you want. I don't think +that it is actually problematic in principle. + +To work around this, I use an import-from-derivation call to get the output +path. The disadvantage is that all the dependencies of your fixed output +derivation will be build in the eval phase. + +## Nix unit tests with nice diffing output + +`nur.repos.kolloch.lib.runTest` exposes `nix-test-runner` runTest. + +Docs are at the [nix-test-runner +repository](https://github.com/stoeffel/nix-test-runner). \ No newline at end of file diff --git a/default.nix b/default.nix index dd3e139..0cb742c 100644 --- a/default.nix +++ b/default.nix @@ -6,14 +6,25 @@ # commands such as: # nix-build -A mypackage -{ pkgs ? import {} }: +{ sources ? import ./nix/sources.nix +, nixpkgs ? sources.nixpkgs +, pkgs ? import nixpkgs {} +, nixTestRunnerSrc ? sources.nix-test-runner +, nixTestRunner ? pkgs.callPackage nixTestRunnerSrc {} +}: -{ +rec { # The `lib`, `modules`, and `overlay` names are special - lib = import ./lib { inherit pkgs; }; # functions + lib = pkgs.callPackage ./lib { inherit nixTestRunner; }; # functions modules = import ./modules; # NixOS modules overlays = import ./overlays; # nixpkgs overlays + tests = pkgs.lib.callPackageWith + (pkgs // { inherit sources; nurKollochLib = lib; } ) + ./tests {}; + + nix-test-runner = nixTestRunner.package; + # Packages. # example-package = pkgs.callPackage ./pkgs/example-package { }; # some-qt5-package = pkgs.libsForQt5.callPackage ./pkgs/some-qt5-package { }; diff --git a/lib/default.nix b/lib/default.nix index 53d6abe..59a04b6 100644 --- a/lib/default.nix +++ b/lib/default.nix @@ -1,8 +1,10 @@ -{ pkgs }: +{ pkgs, lib, nixTestRunner }: -with pkgs.lib; { - # Add your library functions here - # - # hexint = x: hexvals.${toLower x}; +rec { + # I use this to keep individual features also importable independently + # of other code in this NUR repo. + inherit (pkgs.callPackage ./rerun-fixed.nix {}) rerunFixedDerivationOnChange; + + runTests = nixTestRunner.runTests; } diff --git a/lib/rerun-fixed.nix b/lib/rerun-fixed.nix new file mode 100644 index 0000000..def2c66 --- /dev/null +++ b/lib/rerun-fixed.nix @@ -0,0 +1,125 @@ +{ pkgs, lib }: + +rec { + /* Creates a fixed output derivation that reruns if any of its inputs change. + + Type: rerunFixedDerivationOnChange -> derivation -> derivation + + Example: + let myFixedOutputDerivation = pkgs.stdenv.mkDerivation { + # ... + outputHash = "sha256:..."; + outputHashMode = "recursive"; + }; + in + nur.repos.kolloch.lib.rerunFixedDerivationOnChange myFixedOutputDerivation; + + Usually, fixed output derivations are only rerun if their name or hash + changes (i.e. when their output path changes). This makes the derivation + definition irrelevant if the output is in the cache. + + With this helper, you can make sure that the commands that are specified + where at least ran once -- with the exact versions of the buildInputs that + you specified. It does so by putting a hash of all the inputs into the + name. + + Caveat: It uses "import from derivation" under the hood. See implementation + for an explanation why. + + Implementation + + Getting the hash is easy, in principle, because nix already does that work + for us when creating the output paths for non-fixed output derivations. + Therefore, if we create a non-fixed output derivation with the same inputs, + the hash in the output path should change with every change in inputs. + Unfortunately, we cannot use parts of the output path in the name of our + derivation directly. + + Nix prevents this because supposedly it is not what you want. I don't think + that it is actually problematic in principle. + + To work around this, I use an import-from-derivation call to get the output + path. The disadvantage is that all the dependencies of your fixed output + derivation will be build in the eval phase. + */ + rerunFixedDerivationOnChange = fixedOutputDerivation: + assert (lib.assertMsg + (fixedOutputDerivation ? outputHash) + "rerunOnChangedInputs expects a fixedOutputDerivation"); + + let + # The name of the original derivation. + name = fixedOutputDerivation.name or "fixed"; + + # A hash that includes all inputs of the fixedOutputDerivation. + # + # Creates a derivation that has all the same inputs and + # simply outputs it's output path. + # + # If we could simply access the output path from nix + # and use it as a part of the name, this would be much easier. + inputsHash = + let + # Args to rename. + renameArgs = [ "builder" "args" ]; + renamedArgs = + let declash = attrName: + if fixedOutputDerivation ? attrName + then declash "${attrName}_" + else attrName; + copyName = attrName: "original_${declash attrName}"; + copy = attrName: + { + name = copyName attrName; + value = fixedOutputDerivation."${attrName}"; + }; + copies = builtins.map copy renameArgs; + in + builtins.listToAttrs copies; + # Args to replace with null + nullArgs = renameArgs ++ [ "outputHash" "outputHashAlgo" "outputHashMode" ]; + nulledArgs = + let nullMe = name: + { + inherit name; + value = null; + }; + nulled = builtins.map nullMe nullArgs; + in + builtins.listToAttrs nulled; + outputPathBuilderArgs = + assert builtins.isAttrs renamedArgs; + assert builtins.isAttrs nulledArgs; + renamedArgs // nulledArgs // { + name = "outpath-for-${name}"; + builder = "${pkgs.bash}/bin/bash"; + args = ["-c" "set -x; echo $out > $out" ]; + }; + outputPathBuilder = + assert builtins.isAttrs outputPathBuilderArgs; + if fixedOutputDerivation ? overrideAttrs + then fixedOutputDerivation.overrideAttrs (attrs: outputPathBuilderArgs) + else fixedOutputDerivation.overrideDerivation (attrs: outputPathBuilderArgs); + outputPath = + assert lib.isDerivation outputPathBuilder; + builtins.readFile outputPathBuilder; + in builtins.substring 11 32 outputPath; + + # This is the basic idea: Add the inputsHash to the name + # to force a rebuild whenever the inputs change. + nameWithHash = { name = "${inputsHash}_${name}"; }; + + # Make "overrideAttrs" and "overrideDerivation" work as expected. + # That means that we need to keep overriding the name. + overrideAttrs = f: rerunFixedDerivationOnChange (fixedOutputDerivation.overrideAttrs f); + overrideDerivation = f: rerunFixedDerivationOnChange (fixedOutputDerivation.overrideDerivation f); + in + if fixedOutputDerivation ? overrideAttrs + then + (fixedOutputDerivation.overrideAttrs (attrs: nameWithHash)) + // { inherit overrideAttrs overrideDerivation; } + else + (fixedOutputDerivation.overrideDerivation (attrs: nameWithHash)) + // { inherit overrideDerivation; }; +} + diff --git a/nix/sources.json b/nix/sources.json new file mode 100644 index 0000000..a39ea5a --- /dev/null +++ b/nix/sources.json @@ -0,0 +1,38 @@ +{ + "niv": { + "branch": "master", + "description": "Easy dependency management for Nix projects", + "homepage": "https://github.com/nmattia/niv", + "owner": "nmattia", + "repo": "niv", + "rev": "f73bf8d584148677b01859677a63191c31911eae", + "sha256": "0jlmrx633jvqrqlyhlzpvdrnim128gc81q5psz2lpp2af8p8q9qs", + "type": "tarball", + "url": "https://github.com/nmattia/niv/archive/f73bf8d584148677b01859677a63191c31911eae.tar.gz", + "url_template": "https://github.com///archive/.tar.gz" + }, + "nix-test-runner": { + "branch": "master", + "description": "Test runner for nix expressions", + "homepage": "", + "owner": "stoeffel", + "repo": "nix-test-runner", + "rev": "545306ab20a561ef79567e68fa0e25e1ed340ef1", + "sha256": "0iz5kmls6mwwa7lxpi4cq3yq36ygk2vrf0k54xyis736ikx6xky1", + "type": "tarball", + "url": "https://github.com/stoeffel/nix-test-runner/archive/545306ab20a561ef79567e68fa0e25e1ed340ef1.tar.gz", + "url_template": "https://github.com///archive/.tar.gz" + }, + "nixpkgs": { + "branch": "nixos-20.03", + "description": "A read-only mirror of NixOS/nixpkgs tracking the released channels. Send issues and PRs to", + "homepage": "https://github.com/NixOS/nixpkgs", + "owner": "NixOS", + "repo": "nixpkgs-channels", + "rev": "ab3adfe1c769c22b6629e59ea0ef88ec8ee4563f", + "sha256": "1m4wvrrcvif198ssqbdw897c8h84l0cy7q75lyfzdsz9khm1y2n1", + "type": "tarball", + "url": "https://github.com/NixOS/nixpkgs-channels/archive/ab3adfe1c769c22b6629e59ea0ef88ec8ee4563f.tar.gz", + "url_template": "https://github.com///archive/.tar.gz" + } +} diff --git a/nix/sources.nix b/nix/sources.nix new file mode 100644 index 0000000..8a725cb --- /dev/null +++ b/nix/sources.nix @@ -0,0 +1,134 @@ +# This file has been generated by Niv. + +let + + # + # The fetchers. fetch_ fetches specs of type . + # + + fetch_file = pkgs: spec: + if spec.builtin or true then + builtins_fetchurl { inherit (spec) url sha256; } + else + pkgs.fetchurl { inherit (spec) url sha256; }; + + fetch_tarball = pkgs: spec: + if spec.builtin or true then + builtins_fetchTarball { inherit (spec) url sha256; } + else + pkgs.fetchzip { inherit (spec) url sha256; }; + + fetch_git = spec: + builtins.fetchGit { url = spec.repo; inherit (spec) rev ref; }; + + fetch_builtin-tarball = spec: + builtins.trace + '' + WARNING: + The niv type "builtin-tarball" will soon be deprecated. You should + instead use `builtin = true`. + + $ niv modify -a type=tarball -a builtin=true + '' + builtins_fetchTarball { inherit (spec) url sha256; }; + + fetch_builtin-url = spec: + builtins.trace + '' + WARNING: + The niv type "builtin-url" will soon be deprecated. You should + instead use `builtin = true`. + + $ niv modify -a type=file -a builtin=true + '' + (builtins_fetchurl { inherit (spec) url sha256; }); + + # + # Various helpers + # + + # The set of packages used when specs are fetched using non-builtins. + mkPkgs = sources: + let + sourcesNixpkgs = + import (builtins_fetchTarball { inherit (sources.nixpkgs) url sha256; }) {}; + hasNixpkgsPath = builtins.any (x: x.prefix == "nixpkgs") builtins.nixPath; + hasThisAsNixpkgsPath = == ./.; + in + if builtins.hasAttr "nixpkgs" sources + then sourcesNixpkgs + else if hasNixpkgsPath && ! hasThisAsNixpkgsPath then + import {} + else + abort + '' + Please specify either (through -I or NIX_PATH=nixpkgs=...) or + add a package called "nixpkgs" to your sources.json. + ''; + + # The actual fetching function. + fetch = pkgs: name: spec: + + if ! builtins.hasAttr "type" spec then + abort "ERROR: niv spec ${name} does not have a 'type' attribute" + else if spec.type == "file" then fetch_file pkgs spec + else if spec.type == "tarball" then fetch_tarball pkgs spec + else if spec.type == "git" then fetch_git spec + else if spec.type == "builtin-tarball" then fetch_builtin-tarball spec + else if spec.type == "builtin-url" then fetch_builtin-url spec + else + abort "ERROR: niv spec ${name} has unknown type ${builtins.toJSON spec.type}"; + + # Ports of functions for older nix versions + + # a Nix version of mapAttrs if the built-in doesn't exist + mapAttrs = builtins.mapAttrs or ( + f: set: with builtins; + listToAttrs (map (attr: { name = attr; value = f attr set.${attr}; }) (attrNames set)) + ); + + # fetchTarball version that is compatible between all the versions of Nix + builtins_fetchTarball = { url, sha256 }@attrs: + let + inherit (builtins) lessThan nixVersion fetchTarball; + in + if lessThan nixVersion "1.12" then + fetchTarball { inherit url; } + else + fetchTarball attrs; + + # fetchurl version that is compatible between all the versions of Nix + builtins_fetchurl = { url, sha256 }@attrs: + let + inherit (builtins) lessThan nixVersion fetchurl; + in + if lessThan nixVersion "1.12" then + fetchurl { inherit url; } + else + fetchurl attrs; + + # Create the final "sources" from the config + mkSources = config: + mapAttrs ( + name: spec: + if builtins.hasAttr "outPath" spec + then abort + "The values in sources.json should not have an 'outPath' attribute" + else + spec // { outPath = fetch config.pkgs name spec; } + ) config.sources; + + # The "config" used by the fetchers + mkConfig = + { sourcesFile ? ./sources.json + , sources ? builtins.fromJSON (builtins.readFile sourcesFile) + , pkgs ? mkPkgs sources + }: rec { + # The sources, i.e. the attribute set of spec name to spec + inherit sources; + + # The "pkgs" (evaluated nixpkgs) to use for e.g. non-builtin fetchers + inherit pkgs; + }; +in +mkSources (mkConfig {}) // { __functor = _: settings: mkSources (mkConfig settings); } diff --git a/tests/default.nix b/tests/default.nix new file mode 100644 index 0000000..d7e2fc0 --- /dev/null +++ b/tests/default.nix @@ -0,0 +1,75 @@ +{ pkgs, stdenv, lib, nurKollochLib }: + +rec { + # Causes ci.nix to recurse into the tests. + recurseForDerivations = true; + + myFixedOutputDerivation = + pkgs.runCommand + "download-something" + { + buildInputs = [ pkgs.curl pkgs.cacert ]; + outputHash = "sha256:1w9k15dvbawnwn8mqi6v8panf8s7g3p6iaydmkdp87fd7vgxkc14"; + outputHashMode = "recursive"; + } + '' + # This is just a test, obviously, we would use a fetcher for this. + curl -o $out https://static.crates.io/crates/nix-base32/nix-base32-0.1.1.crate + ''; + + myChangedFixedOutputDerivation = + pkgs.runCommand + "download-something" + { + buildInputs = [ pkgs.curl pkgs.cacert ]; + outputHash = "sha256:1w9k15dvbawnwn8mqi6v8panf8s7g3p6iaydmkdp87fd7vgxkc14"; + outputHashMode = "recursive"; + } + '' + # TOTALLY DIFFERENT + curl -o $out https://static.crates.io/crates/nix-base32/nix-base32-0.1.1.crate + ''; + + + myOverriddenFixedOutputDerivation = myFixedOutputDerivation.overrideAttrs (attrs: { + anotherAttribute = "overridden"; + }); + + rerunOnChange_myFixedOutputDerivation = + nurKollochLib.rerunFixedDerivationOnChange myFixedOutputDerivation; + + rerunOnChange_myChangedFixedOutputDerivation = + nurKollochLib.rerunFixedDerivationOnChange myChangedFixedOutputDerivation; + + overridden_rerunOnChange_myFixedOutputDerivation = + rerunOnChange_myFixedOutputDerivation.overrideAttrs (attrs: { something = "change"; }); + + rerunFixedDerivationOnChangeTests = nurKollochLib.runTests { + name = "rerunFixedDerivationOnChange"; + tests = { + testChangedFixedOutputDerivation = { + expr = "${myFixedOutputDerivation}"; + expected = "${myChangedFixedOutputDerivation}"; + }; + testOverriddenFixedOutputDerivation = { + expr = "${myFixedOutputDerivation}"; + expected = "${myOverriddenFixedOutputDerivation}"; + }; + + testInstrumentedFixedOutputDerivation = { + expr = + if "${rerunOnChange_myFixedOutputDerivation}" != "${rerunOnChange_myChangedFixedOutputDerivation}" + then "" + else "both paths are the same: ${rerunOnChange_myFixedOutputDerivation}"; + expected = ""; + }; + testOverrideResultsInChange = { + expr = + if "${rerunOnChange_myFixedOutputDerivation}" != "${overridden_rerunOnChange_myFixedOutputDerivation}" + then "" + else "both paths are the same: ${rerunOnChange_myFixedOutputDerivation}"; + expected = ""; + }; + }; + }; +} \ No newline at end of file