monitoring: grafana and prometheus service enabled

This enables the foundations for grafana and prometheus services on the
monitoring vm. Currently we still need to:

  - Enable TLS
  - Generate a static configs for scapers
  - Enable prometheus exporters on Servers
  - Enable prometheus exporters on APs
  - Enable prometheus exporters on Pis

This will be done in follow up PRs.

Prometheus is also added as a common nixos module so that it can be
consumed from other machines.

nix/machines/_common/prometheus.nix

@@ -0,0 +1,20 @@
+{ ... }:
+let
+  port = 9100;
+in
+{
+  networking.firewall.allowedTCPPorts = [ port ];
+
+  services.prometheus.exporters.node = {
+    enable = true;
+    port = port;
+    enabledCollectors = [
+      "logind"
+      "systemd"
+      "network_route"
+    ];
+    disabledCollectors = [
+      "textfile"
+    ];
+  };
+}

nix/machines/flake-module.nix

@@ -21,6 +21,14 @@ in
         ];
         specialArgs = { inherit inputs; };
       };
+      monitor = lib.nixosSystem {
+        inherit system;
+        modules = [
+          common
+          ./monitor.nix
+        ];
+        specialArgs = { inherit inputs; };
+      };
       massflash = lib.nixosSystem {
         inherit system;
         modules = [

nix/machines/monitor.nix

@@ -0,0 +1,88 @@
+{ config, lib, pkgs, ... }:
+let
+  hostname = "monitoring.scale.lan";
+in
+{
+  imports =
+    [
+      ./_common/prometheus.nix
+    ];
+
+  # If not present then warning and will be set to latest release during build
+  system.stateVersion = "23.05";
+
+  boot.kernelParams = [ "console=ttyS0" "boot.shell_on_fail" ];
+
+  networking.firewall.allowedTCPPorts = [ 80 443 ];
+
+  # TODO: How to handle sudo esculation
+  security.sudo.wheelNeedsPassword = false;
+
+  environment.systemPackages = with pkgs; [
+    vim
+    git
+    bintools
+  ];
+
+  services = {
+    openssh = {
+      enable = true;
+    };
+
+    prometheus = {
+      enable = true;
+      enableReload = true;
+      scrapeConfigs = [
+        {
+          job_name = "prometheus";
+          static_configs = [
+            {
+              targets = [ "localhost:${toString config.services.prometheus.exporters.node.port}" ];
+              labels = { instance = "localhost"; };
+            }
+          ];
+        }
+      ];
+    };
+
+    grafana = {
+      enable = true;
+      settings = {
+        server = {
+          http_addr = "127.0.0.1";
+          http_port = 3000;
+          domain = "${hostname}";
+        };
+        analytics.reporting_enabled = false;
+      };
+      provision = {
+        # Can use just datasources anymore
+        # https://github.com/NixOS/nixpkgs/blob/41de143fda10e33be0f47eab2bfe08a50f234267/nixos/modules/services/monitoring/grafana.nix#L101-L104
+        datasources.settings.datasources = [
+          {
+            name = "prometheus";
+            type = "prometheus";
+            access = "proxy";
+            url = "http://127.0.0.1:${toString config.services.prometheus.port}";
+          }
+        ];
+      };
+    };
+
+    nginx = {
+      enable = true;
+      # TODO: TLS enabled
+      # Good example enable TLS, but would like to keep it out of the /nix/store
+      # ref: https://github.com/NixOS/nixpkgs/blob/c6fd903606866634312e40cceb2caee8c0c9243f/nixos/tests/custom-ca.nix#L80
+      virtualHosts."${hostname}" = {
+        default = true;
+        # ACME wont work for us on the private network
+        enableACME = false;
+        locations."/" = {
+          proxyPass = "http://${toString config.services.grafana.settings.server.http_addr}:${toString config.services.grafana.settings.server.http_port}/";
+          proxyWebsockets = true;
+        };
+      };
+    };
+  };
+}