switch-to-configuration.pl: Fix timers resulting in old unit running

[nh2]

Fixes #49415.

See the added comment for a detail description.

See also the related #49528.

Things done

• Built on platform(s)
  • x86_64-linux
  • aarch64-linux
  • x86_64-darwin
  • aarch64-darwin
• For non-Linux: Is sandboxing enabled in nix.conf? (See Nix manual)
  • sandbox = relaxed
  • sandbox = true
• Tested, as applicable:
  • NixOS test(s) (look inside nixos/tests)
  • and/or package tests
  • or, for functions and "core" functionality, tests in lib/tests or pkgs/test
  • made sure NixOS tests are linked to the relevant packages
• Tested compilation of all packages that depend on this change using nix-shell -p nixpkgs-review --run "nixpkgs-review rev HEAD". Note: all changes have to be committed, also see nixpkgs-review usage
• Tested basic functionality of all binary files (usually in ./result/bin/)
• 24.05 Release Notes (or backporting 23.05 and 23.11 Release notes)
  • (Package updates) Added a release notes entry if the change is major or breaking
  • (Module updates) Added a release notes entry if the change is significant
  • (Module addition) Added a release notes entry if adding a new NixOS module
• Fits CONTRIBUTING.md.

---

Add a 👍 reaction to pull requests you find important.

CC:

• @flokli @arianvp @Gabriella439 from systemd timers can result in changed services not being restarted reliably #49415.
• @symphorien from systemd: Interactions of restartIfChanged, reloadIfChanged, stopIfChanged is fragile #49528 (comment)
• @dasJ from systemd: Interactions of restartIfChanged, reloadIfChanged, stopIfChanged is fragile #49528

[Mic92]

Is this not potentially a quite long list. I thought this is what the filter above was for?

[nh2]

It's a medium long list on my servers, e.g. 3-4 lines in my 100-char terminal.

I believe it is worth showing, because:

• It's useful for debugging what's going on. The systemctl start done here is one of the 3 key actions done in configuration switching (the other ones being stop and restart), and depending on what your units/targets do, it may be the last line you see before your server loses the network or desktop loses the GPU output. In those cases, it can be extremely helpful to see what's being started.
• This not being an invisble start invocation will hopefully reduce the number of issues people can't diagnose/solve by themselves, and thus number of issues they file. Most NixOS users understand systemctl and can reproduce issues when being told what systemctl commands to run, but much fewer users are able/willing to read/modify Perl to be able see the invocation.

I added it as a separate log line in addition to the previous one that lists only the stopped units, so that the short, most important line stays easy to read for the user. Then the longer line follows, which the user can ignore if their system didn't go down.

[arianvp]

But this stopping units that are getting removed from the NixOS configuration right? So this specific case here is probably very rare (I hope)

[arianvp]

Wouldn't changing start to a restart cause ExecStop to run twice? Once in the old generation through the systemctl stop and once in the new generation through the systemctl restart. This seems like surprising behaviour to me and I wonder if it will break things subtly

[arianvp]

Oh: but if it's stopped already I guess it isn't stopped a second time by restart... Okay perhaps this approach actually works

[symphorien]

Since this is a tricky problem I wrote a regression test, feel free to include it

import ./make-test-python.nix ({ pkgs, ... }: {
  name = "nixos-rebuild-timers";

  nodes = {
    machine = { lib, pkgs, ... }: {
      imports = [
        ../modules/profiles/installation-device.nix
        ../modules/profiles/base.nix
      ];

      nix.settings = {
        substituters = lib.mkForce [ ];
        hashed-mirrors = null;
        connect-timeout = 1;
      };

      system.includeBuildDependencies = true;

      system.extraDependencies = [
        # Not part of the initial build apparently?
        pkgs.grub2
      ];

      virtualisation = {
        cores = 2;
        memorySize = 4096;
      };

      systemd.services.dostuff = {
        script = "while true; do echo a > /tmp/a; sleep 1; done";
        startAt = "*:*"; # every second, in practice it seems rate limited to 30s
        serviceConfig.Type="simple";
      };

      # this service ensures that switching takes at least 60s, enough time for
      # dostuff.timer to fire
      systemd.services.delaySwitch = {
        script = "echo delaySwitch started";
        postStop = "echo start sleeping; sleep 60; echo stop sleeping";
        serviceConfig.Type="oneshot";
        serviceConfig.RemainAfterExit=true;
        wantedBy = [ "multi-user.target" ];
      };
    };
  };

  testScript =
    let
      configFile = pkgs.writeText "configuration.nix" ''
        { lib, pkgs, ... }: {
          imports = [
            ./hardware-configuration.nix
            <nixpkgs/nixos/modules/testing/test-instrumentation.nix>
          ];

          boot.loader.grub = {
            enable = true;
            device = "/dev/vda";
            forceInstall = true;
          };

          documentation.enable = false;

      systemd.services.dostuff = {
        script = "echo b > /tmp/a";
        startAt = "*:*"; # every second
        serviceConfig.Type="oneshot";
      };

      systemd.services.delaySwitch = {
        script = "echo delaySwitch changed";
        postStop = "echo start sleeping; sleep 60; echo stop sleeping";
        serviceConfig.Type="oneshot";
        serviceConfig.RemainAfterExit=true;
        wantedBy = [ "multi-user.target" ];
      };

        }
      '';

    in
    ''
      machine.start()
      machine.succeed("udevadm settle")
      machine.wait_for_unit("multi-user.target")

      machine.succeed("nixos-generate-config")
      machine.copy_from_host(
          "${configFile}",
          "/etc/nixos/configuration.nix",
      )

      with subtest("Switch to the base system"):
          machine.succeed("nixos-rebuild switch")

      machine.wait_until_succeeds("grep b /tmp/a")

    '';
})