From 0f1d12cb34c1b0c375141973a5ccdf81ebbb58f9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Motiejus=20Jak=C5=A1tys?= <motiejus@jakstys.lt>
Date: Thu, 20 Jul 2023 09:31:26 +0300
Subject: [PATCH] unitstatus: pre-defined service units now exist

---
 hosts/hel1-a/configuration.nix      | 148 +++++++++++-----------------
 modules/base/unitstatus/default.nix |  82 +++++++--------
 modules/base/zfsborg/default.nix    |  64 +++++++-----
 3 files changed, 130 insertions(+), 164 deletions(-)

diff --git a/hosts/hel1-a/configuration.nix b/hosts/hel1-a/configuration.nix
index 7e37b98..1ed30ce 100644
--- a/hosts/hel1-a/configuration.nix
+++ b/hosts/hel1-a/configuration.nix
@@ -6,29 +6,6 @@
   myData,
   ...
 }: let
-  backup_paths = {
-    var_lib = {
-      mountpoint = "/var/lib";
-      zfs_name = "rpool/nixos/var/lib";
-      paths = [
-        "/var/lib/.snapshot-latest/gitea"
-        "/var/lib/.snapshot-latest/headscale"
-        "/var/lib/.snapshot-latest/matrix-synapse"
-      ];
-      backup_at = "*-*-* 00:11:00";
-    };
-    var_log = {
-      mountpoint = "/var/log";
-      zfs_name = "rpool/nixos/var/log";
-      paths = ["/var/log/.snapshot-latest/caddy/"];
-      patterns = [
-        "+ /var/log/.snapshot-latest/caddy/access-jakstys.lt.log-*.zst"
-        "- *"
-      ];
-      backup_at = "*-*-* 00:10:00";
-    };
-  };
-
   turn_cert_dir = "/var/lib/caddy/.local/share/caddy/certificates/acme-v02.api.letsencrypt.org-directory/turn.jakstys.lt";
   gitea_uidgid = 995;
 
@@ -106,7 +83,7 @@ in {
         enable = true;
         email = "motiejus+alerts@jakstys.lt";
         # see TODO in base/unitstatus/default.nix
-        #units = ["zfs-scrub"];
+        units = ["zfs-scrub" "nixos-upgrade"];
       };
     };
   };
@@ -531,79 +508,68 @@ in {
     "d /run/matrix-synapse 0700 matrix-synapse matrix-synapse -"
   ];
 
-  systemd.services =
-    {
-      coturn = {
-        preStart = ''
-          ln -sf ''${CREDENTIALS_DIRECTORY}/tls-key.pem /run/coturn/tls-key.pem
-          ln -sf ''${CREDENTIALS_DIRECTORY}/tls-cert.pem /run/coturn/tls-cert.pem
-        '';
-        unitConfig.ConditionPathExists = [
-          "${turn_cert_dir}/turn.jakstys.lt.key"
-          "${turn_cert_dir}/turn.jakstys.lt.crt"
-        ];
-        serviceConfig.LoadCredential = [
-          "static-auth-secret:${config.age.secrets.turn-static-auth-secret.path}"
-          "tls-key.pem:${turn_cert_dir}/turn.jakstys.lt.key"
-          "tls-cert.pem:${turn_cert_dir}/turn.jakstys.lt.crt"
-        ];
-      };
+  systemd.services = {
+    coturn = {
+      preStart = ''
+        ln -sf ''${CREDENTIALS_DIRECTORY}/tls-key.pem /run/coturn/tls-key.pem
+        ln -sf ''${CREDENTIALS_DIRECTORY}/tls-cert.pem /run/coturn/tls-cert.pem
+      '';
+      unitConfig.ConditionPathExists = [
+        "${turn_cert_dir}/turn.jakstys.lt.key"
+        "${turn_cert_dir}/turn.jakstys.lt.crt"
+      ];
+      serviceConfig.LoadCredential = [
+        "static-auth-secret:${config.age.secrets.turn-static-auth-secret.path}"
+        "tls-key.pem:${turn_cert_dir}/turn.jakstys.lt.key"
+        "tls-cert.pem:${turn_cert_dir}/turn.jakstys.lt.crt"
+      ];
+    };
 
-      headscale = {
-        unitConfig.StartLimitIntervalSec = "5m";
+    headscale = {
+      unitConfig.StartLimitIntervalSec = "5m";
 
-        # Allow restarts for up to a minute. A start
-        # itself may take a while, thus the window of restart
-        # is higher.
-        unitConfig.StartLimitBurst = 50;
-        serviceConfig.RestartSec = 1;
-      };
+      # Allow restarts for up to a minute. A start
+      # itself may take a while, thus the window of restart
+      # is higher.
+      unitConfig.StartLimitBurst = 50;
+      serviceConfig.RestartSec = 1;
+    };
 
-      matrix-synapse = let
-        # TODO https://github.com/NixOS/nixpkgs/pull/222336 replace with `preStart`
-        secretsScript = pkgs.writeShellScript "write-secrets" ''
-          set -euo pipefail
-          umask 077
-          ln -sf ''${CREDENTIALS_DIRECTORY}/jakstys_lt_signing_key /run/matrix-synapse/jakstys_lt_signing_key
-          cat > /run/matrix-synapse/secrets.yaml <<EOF
-          registration_shared_secret: "$(cat ''${CREDENTIALS_DIRECTORY}/registration_shared_secret)"
-          macaroon_secret_key: "$(cat ''${CREDENTIALS_DIRECTORY}/macaroon_secret_key)"
-          turn_shared_secret: "$(cat ''${CREDENTIALS_DIRECTORY}/turn_shared_secret)"
-          EOF
-        '';
-      in {
-        serviceConfig.ExecStartPre = ["" secretsScript];
-        serviceConfig.LoadCredential = [
-          "jakstys_lt_signing_key:${config.age.secrets.synapse-jakstys-signing-key.path}"
-          "registration_shared_secret:${config.age.secrets.synapse-registration-shared-secret.path}"
-          "macaroon_secret_key:${config.age.secrets.synapse-macaroon-secret-key.path}"
-          "turn_shared_secret:${config.age.secrets.turn-static-auth-secret.path}"
-        ];
-      };
+    matrix-synapse = let
+      # TODO https://github.com/NixOS/nixpkgs/pull/222336 replace with `preStart`
+      secretsScript = pkgs.writeShellScript "write-secrets" ''
+        set -euo pipefail
+        umask 077
+        ln -sf ''${CREDENTIALS_DIRECTORY}/jakstys_lt_signing_key /run/matrix-synapse/jakstys_lt_signing_key
+        cat > /run/matrix-synapse/secrets.yaml <<EOF
+        registration_shared_secret: "$(cat ''${CREDENTIALS_DIRECTORY}/registration_shared_secret)"
+        macaroon_secret_key: "$(cat ''${CREDENTIALS_DIRECTORY}/macaroon_secret_key)"
+        turn_shared_secret: "$(cat ''${CREDENTIALS_DIRECTORY}/turn_shared_secret)"
+        EOF
+      '';
+    in {
+      serviceConfig.ExecStartPre = ["" secretsScript];
+      serviceConfig.LoadCredential = [
+        "jakstys_lt_signing_key:${config.age.secrets.synapse-jakstys-signing-key.path}"
+        "registration_shared_secret:${config.age.secrets.synapse-registration-shared-secret.path}"
+        "macaroon_secret_key:${config.age.secrets.synapse-macaroon-secret-key.path}"
+        "turn_shared_secret:${config.age.secrets.turn-static-auth-secret.path}"
+      ];
+    };
 
-      cert-watcher = {
-        description = "Restart coturn when tls key/cert changes";
-        wantedBy = ["multi-user.target"];
-        unitConfig = {
-          StartLimitIntervalSec = 10;
-          StartLimitBurst = 5;
-        };
-        serviceConfig = {
-          Type = "oneshot";
-          ExecStart = "${pkgs.systemd}/bin/systemctl restart coturn.service";
-        };
+    cert-watcher = {
+      description = "Restart coturn when tls key/cert changes";
+      wantedBy = ["multi-user.target"];
+      unitConfig = {
+        StartLimitIntervalSec = 10;
+        StartLimitBurst = 5;
       };
-
-      zfs-scrub.unitConfig.OnFailure = "unit-status-mail@zfs-scrub.service";
-      nixos-upgrade.unitConfig.OnFailure = "unit-status-mail@nixos-upgrade.service";
-    }
-    // lib.mapAttrs' (name: value: {
-      name = "borgbackup-job-${name}";
-      value = {
-        unitConfig.OnFailure = "unit-status-mail@borgbackup-job-${name}.service";
+      serviceConfig = {
+        Type = "oneshot";
+        ExecStart = "${pkgs.systemd}/bin/systemctl restart coturn.service";
       };
-    })
-    backup_paths;
+    };
+  };
 
   systemd.paths = {
     cert-watcher = {
diff --git a/modules/base/unitstatus/default.nix b/modules/base/unitstatus/default.nix
index 1191b92..76e1e4a 100644
--- a/modules/base/unitstatus/default.nix
+++ b/modules/base/unitstatus/default.nix
@@ -5,60 +5,50 @@
   ...
 }: {
   # TODO:
-  # - accept unit names:
-  #   - assert they exist
-  #   - add 'systemd.<unit>.unitConfig.OnFailure' to point to this one.
   # - assert postfix is configured
   options.mj.base.unitstatus = with lib.types; {
     enable = lib.mkEnableOption "alert by email on unit failure";
     email = lib.mkOption {type = str;};
-    #units = lib.mkOption {type = lisOf str;};
+    units = lib.mkOption {type = listOf str;};
   };
 
-  config =
-    lib.mkIf config.mj.base.unitstatus.enable {
-      systemd.services."unit-status-mail@" = let
-        # https://northernlightlabs.se/2014-07-05/systemd-status-mail-on-unit-failure.html
-        script = pkgs.writeShellScript "unit-status-mail" ''
-          set -e
-          MAILTO="${config.mj.base.unitstatus.email}"
-          UNIT=$1
-          EXTRA=""
-          for e in "''${@:2}"; do
-            EXTRA+="$e"$'\n'
-          done
-          UNITSTATUS=$(${pkgs.systemd}/bin/systemctl status "$UNIT")
-          ${pkgs.postfix}/bin/sendmail $MAILTO <<EOF
-          Subject:Status mail for unit: $UNIT
+  config = lib.mkIf config.mj.base.unitstatus.enable {
+    systemd.services =
+      {
+        "unit-status-mail@" = let
+          # https://northernlightlabs.se/2014-07-05/systemd-status-mail-on-unit-failure.html
+          script = pkgs.writeShellScript "unit-status-mail" ''
+            set -e
+            MAILTO="${config.mj.base.unitstatus.email}"
+            UNIT=$1
+            EXTRA=""
+            for e in "''${@:2}"; do
+              EXTRA+="$e"$'\n'
+            done
+            UNITSTATUS=$(${pkgs.systemd}/bin/systemctl status "$UNIT")
+            ${pkgs.postfix}/bin/sendmail $MAILTO <<EOF
+            Subject:Status mail for unit: $UNIT
 
-          Status report for unit: $UNIT
-          $EXTRA
+            Status report for unit: $UNIT
+            $EXTRA
 
-          $UNITSTATUS
-          EOF
+            $UNITSTATUS
+            EOF
 
-          echo -e "Status mail sent to: $MAILTO for unit: $UNIT"
-        '';
-      in {
-        description = "Send an email on unit failure";
-        serviceConfig = {
-          Type = "simple";
-          ExecStart = ''${script} "%I" "Hostname: %H" "Machine ID: %m" "Boot ID: %b" '';
+            echo -e "Status mail sent to: $MAILTO for unit: $UNIT"
+          '';
+        in {
+          description = "Send an email on unit failure";
+          serviceConfig = {
+            Type = "simple";
+            ExecStart = ''${script} "%I" "Hostname: %H" "Machine ID: %m" "Boot ID: %b" '';
+          };
         };
-      };
-    #};
-    # See TODO above.
-    #// {
-    #  systemd.services =
-    #    lib.listToAttrs
-    #    (map (
-    #        unit: {
-    #          name = unit;
-    #          value = {
-    #            unitConfig = {OnFailure = "unit-status-mail@${unit}.service";};
-    #          };
-    #        }
-    #      )
-    #      config.mj.base.unitstatus.units);
-    };
+      }
+      // lib.genAttrs config.mj.base.unitstatus.units (
+        unit: {
+          unitConfig = {OnFailure = "unit-status-mail@${unit}.service";};
+        }
+      );
+  };
 }
diff --git a/modules/base/zfsborg/default.nix b/modules/base/zfsborg/default.nix
index 198537e..3c41bff 100644
--- a/modules/base/zfsborg/default.nix
+++ b/modules/base/zfsborg/default.nix
@@ -57,34 +57,44 @@ in {
     services.borgbackup.jobs = lib.mapAttrs' (mountpoint: attrs: let
       fs = builtins.getAttr mountpoint config.fileSystems;
     in
-      assert fs.fsType == "zfs"; {
-        name = lib.strings.sanitizeDerivationName mountpoint;
-        value =
-          {
-            doInit = true;
-            repo = config.mj.base.zfsborg.repo;
-            encryption = {
-              mode = "repokey-blake2";
-              passCommand = "cat ${config.mj.base.zfsborg.passwdPath}";
+      assert fs.fsType == "zfs";
+      assert lib.assertMsg
+          config.mj.base.unitstatus.enable
+          "config.mj.base.unitstatus.enable must be true";
+        {
+          name = lib.strings.sanitizeDerivationName mountpoint;
+          value =
+            {
+              doInit = true;
+              repo = config.mj.base.zfsborg.repo;
+              encryption = {
+                mode = "repokey-blake2";
+                passCommand = "cat ${config.mj.base.zfsborg.passwdPath}";
+              };
+              paths = attrs.paths;
+              extraArgs = "--remote-path=borg1";
+              compression = "auto,lzma";
+              startAt = attrs.backup_at;
+              readWritePaths = let p = mountpoint + "/.snapshot-latest"; in [p];
+              preHook = mountLatest mountpoint fs.device;
+              postHook = umountLatest mountpoint;
+              prune.keep = {
+                within = "1d";
+                daily = 7;
+                weekly = 4;
+                monthly = 3;
+              };
+            }
+            // lib.optionalAttrs (attrs ? patterns) {
+              patterns = attrs.patterns;
             };
-            paths = attrs.paths;
-            extraArgs = "--remote-path=borg1";
-            compression = "auto,lzma";
-            startAt = attrs.backup_at;
-            readWritePaths = let p = mountpoint + "/.snapshot-latest"; in [p];
-            preHook = mountLatest mountpoint fs.device;
-            postHook = umountLatest mountpoint;
-            prune.keep = {
-              within = "1d";
-              daily = 7;
-              weekly = 4;
-              monthly = 3;
-            };
-          }
-          // lib.optionalAttrs (attrs ? patterns) {
-            patterns = attrs.patterns;
-          };
-      })
+        })
     config.mj.base.zfsborg.mountpoints;
+
+    mj.base.unitstatus.units = let
+      mounts = config.mj.base.zfsborg.mountpoints;
+      sanitized = map lib.strings.sanitizeDerivationName (lib.attrNames mounts);
+    in
+      map (n: "borgbackup-job-${n}") sanitized;
   };
 }