motiejus
/
config
1
Fork 0
Code Packages Releases Activity

unitstatus: pre-defined service units now exist

This commit is contained in:
Motiejus Jakštys 2023-07-20 09:31:26 +03:00
parent 1bfd201028
commit 0f1d12cb34
3 changed files with 130 additions and 164 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

148
hosts/hel1-a/configuration.nix
View File

@ -6,29 +6,6 @@
myData, myData,
... ...
}: let }: let
backup_paths = {
var_lib = {
mountpoint = "/var/lib";
zfs_name = "rpool/nixos/var/lib";
paths = [
"/var/lib/.snapshot-latest/gitea"
"/var/lib/.snapshot-latest/headscale"
"/var/lib/.snapshot-latest/matrix-synapse"
];
backup_at = "*-*-* 00:11:00";
};
var_log = {
mountpoint = "/var/log";
zfs_name = "rpool/nixos/var/log";
paths = ["/var/log/.snapshot-latest/caddy/"];
patterns = [
"+ /var/log/.snapshot-latest/caddy/access-jakstys.lt.log-*.zst"
"- *"
];
backup_at = "*-*-* 00:10:00";
};
};
turn_cert_dir = "/var/lib/caddy/.local/share/caddy/certificates/acme-v02.api.letsencrypt.org-directory/turn.jakstys.lt"; turn_cert_dir = "/var/lib/caddy/.local/share/caddy/certificates/acme-v02.api.letsencrypt.org-directory/turn.jakstys.lt";
gitea_uidgid = 995; gitea_uidgid = 995;
@ -106,7 +83,7 @@ in {
enable = true; enable = true;
email = "motiejus+alerts@jakstys.lt"; email = "motiejus+alerts@jakstys.lt";
# see TODO in base/unitstatus/default.nix # see TODO in base/unitstatus/default.nix
#units = ["zfs-scrub"]; units = ["zfs-scrub" "nixos-upgrade"];
}; };
}; };
}; };
@ -531,79 +508,68 @@ in {
"d /run/matrix-synapse 0700 matrix-synapse matrix-synapse -" "d /run/matrix-synapse 0700 matrix-synapse matrix-synapse -"
]; ];
systemd.services = systemd.services = {
{ coturn = {
coturn = { preStart = ''
preStart = '' ln -sf ''${CREDENTIALS_DIRECTORY}/tls-key.pem /run/coturn/tls-key.pem
ln -sf ''${CREDENTIALS_DIRECTORY}/tls-key.pem /run/coturn/tls-key.pem ln -sf ''${CREDENTIALS_DIRECTORY}/tls-cert.pem /run/coturn/tls-cert.pem
ln -sf ''${CREDENTIALS_DIRECTORY}/tls-cert.pem /run/coturn/tls-cert.pem '';
''; unitConfig.ConditionPathExists = [
unitConfig.ConditionPathExists = [ "${turn_cert_dir}/turn.jakstys.lt.key"
"${turn_cert_dir}/turn.jakstys.lt.key" "${turn_cert_dir}/turn.jakstys.lt.crt"
"${turn_cert_dir}/turn.jakstys.lt.crt" ];
]; serviceConfig.LoadCredential = [
serviceConfig.LoadCredential = [ "static-auth-secret:${config.age.secrets.turn-static-auth-secret.path}"
"static-auth-secret:${config.age.secrets.turn-static-auth-secret.path}" "tls-key.pem:${turn_cert_dir}/turn.jakstys.lt.key"
"tls-key.pem:${turn_cert_dir}/turn.jakstys.lt.key" "tls-cert.pem:${turn_cert_dir}/turn.jakstys.lt.crt"
"tls-cert.pem:${turn_cert_dir}/turn.jakstys.lt.crt" ];
]; };
};
headscale = { headscale = {
unitConfig.StartLimitIntervalSec = "5m"; unitConfig.StartLimitIntervalSec = "5m";
# Allow restarts for up to a minute. A start # Allow restarts for up to a minute. A start
# itself may take a while, thus the window of restart # itself may take a while, thus the window of restart
# is higher. # is higher.
unitConfig.StartLimitBurst = 50; unitConfig.StartLimitBurst = 50;
serviceConfig.RestartSec = 1; serviceConfig.RestartSec = 1;
}; };
matrix-synapse = let matrix-synapse = let
# TODO https://github.com/NixOS/nixpkgs/pull/222336 replace with `preStart` # TODO https://github.com/NixOS/nixpkgs/pull/222336 replace with `preStart`
secretsScript = pkgs.writeShellScript "write-secrets" '' secretsScript = pkgs.writeShellScript "write-secrets" ''
set -euo pipefail set -euo pipefail
umask 077 umask 077
ln -sf ''${CREDENTIALS_DIRECTORY}/jakstys_lt_signing_key /run/matrix-synapse/jakstys_lt_signing_key ln -sf ''${CREDENTIALS_DIRECTORY}/jakstys_lt_signing_key /run/matrix-synapse/jakstys_lt_signing_key
cat > /run/matrix-synapse/secrets.yaml <<EOF cat > /run/matrix-synapse/secrets.yaml <<EOF
registration_shared_secret: "$(cat ''${CREDENTIALS_DIRECTORY}/registration_shared_secret)" registration_shared_secret: "$(cat ''${CREDENTIALS_DIRECTORY}/registration_shared_secret)"
macaroon_secret_key: "$(cat ''${CREDENTIALS_DIRECTORY}/macaroon_secret_key)" macaroon_secret_key: "$(cat ''${CREDENTIALS_DIRECTORY}/macaroon_secret_key)"
turn_shared_secret: "$(cat ''${CREDENTIALS_DIRECTORY}/turn_shared_secret)" turn_shared_secret: "$(cat ''${CREDENTIALS_DIRECTORY}/turn_shared_secret)"
EOF EOF
''; '';
in { in {
serviceConfig.ExecStartPre = ["" secretsScript]; serviceConfig.ExecStartPre = ["" secretsScript];
serviceConfig.LoadCredential = [ serviceConfig.LoadCredential = [
"jakstys_lt_signing_key:${config.age.secrets.synapse-jakstys-signing-key.path}" "jakstys_lt_signing_key:${config.age.secrets.synapse-jakstys-signing-key.path}"
"registration_shared_secret:${config.age.secrets.synapse-registration-shared-secret.path}" "registration_shared_secret:${config.age.secrets.synapse-registration-shared-secret.path}"
"macaroon_secret_key:${config.age.secrets.synapse-macaroon-secret-key.path}" "macaroon_secret_key:${config.age.secrets.synapse-macaroon-secret-key.path}"
"turn_shared_secret:${config.age.secrets.turn-static-auth-secret.path}" "turn_shared_secret:${config.age.secrets.turn-static-auth-secret.path}"
]; ];
}; };
cert-watcher = { cert-watcher = {
description = "Restart coturn when tls key/cert changes"; description = "Restart coturn when tls key/cert changes";
wantedBy = ["multi-user.target"]; wantedBy = ["multi-user.target"];
unitConfig = { unitConfig = {
StartLimitIntervalSec = 10; StartLimitIntervalSec = 10;
StartLimitBurst = 5; StartLimitBurst = 5;
};
serviceConfig = {
Type = "oneshot";
ExecStart = "${pkgs.systemd}/bin/systemctl restart coturn.service";
};
}; };
serviceConfig = {
zfs-scrub.unitConfig.OnFailure = "unit-status-mail@zfs-scrub.service"; Type = "oneshot";
nixos-upgrade.unitConfig.OnFailure = "unit-status-mail@nixos-upgrade.service"; ExecStart = "${pkgs.systemd}/bin/systemctl restart coturn.service";
}
// lib.mapAttrs' (name: value: {
name = "borgbackup-job-${name}";
value = {
unitConfig.OnFailure = "unit-status-mail@borgbackup-job-${name}.service";
}; };
}) };
backup_paths; };
systemd.paths = { systemd.paths = {
cert-watcher = { cert-watcher = {

82
modules/base/unitstatus/default.nix
View File

@ -5,60 +5,50 @@
... ...
}: { }: {
# TODO: # TODO:
# - accept unit names:
# - assert they exist
# - add 'systemd.<unit>.unitConfig.OnFailure' to point to this one.
# - assert postfix is configured # - assert postfix is configured
options.mj.base.unitstatus = with lib.types; { options.mj.base.unitstatus = with lib.types; {
enable = lib.mkEnableOption "alert by email on unit failure"; enable = lib.mkEnableOption "alert by email on unit failure";
email = lib.mkOption {type = str;}; email = lib.mkOption {type = str;};
#units = lib.mkOption {type = lisOf str;}; units = lib.mkOption {type = listOf str;};
}; };
config = config = lib.mkIf config.mj.base.unitstatus.enable {
lib.mkIf config.mj.base.unitstatus.enable { systemd.services =
systemd.services."unit-status-mail@" = let {
# https://northernlightlabs.se/2014-07-05/systemd-status-mail-on-unit-failure.html "unit-status-mail@" = let
script = pkgs.writeShellScript "unit-status-mail" '' # https://northernlightlabs.se/2014-07-05/systemd-status-mail-on-unit-failure.html
set -e script = pkgs.writeShellScript "unit-status-mail" ''
MAILTO="${config.mj.base.unitstatus.email}" set -e
UNIT=$1 MAILTO="${config.mj.base.unitstatus.email}"
EXTRA="" UNIT=$1
for e in "''${@:2}"; do EXTRA=""
EXTRA+="$e"$'\n' for e in "''${@:2}"; do
done EXTRA+="$e"$'\n'
UNITSTATUS=$(${pkgs.systemd}/bin/systemctl status "$UNIT") done
${pkgs.postfix}/bin/sendmail $MAILTO <<EOF UNITSTATUS=$(${pkgs.systemd}/bin/systemctl status "$UNIT")
Subject:Status mail for unit: $UNIT ${pkgs.postfix}/bin/sendmail $MAILTO <<EOF
Subject:Status mail for unit: $UNIT
Status report for unit: $UNIT Status report for unit: $UNIT
$EXTRA $EXTRA
$UNITSTATUS $UNITSTATUS
EOF EOF
echo -e "Status mail sent to: $MAILTO for unit: $UNIT" echo -e "Status mail sent to: $MAILTO for unit: $UNIT"
''; '';
in { in {
description = "Send an email on unit failure"; description = "Send an email on unit failure";
serviceConfig = { serviceConfig = {
Type = "simple"; Type = "simple";
ExecStart = ''${script} "%I" "Hostname: %H" "Machine ID: %m" "Boot ID: %b" ''; ExecStart = ''${script} "%I" "Hostname: %H" "Machine ID: %m" "Boot ID: %b" '';
};
}; };
}; }
#}; // lib.genAttrs config.mj.base.unitstatus.units (
# See TODO above. unit: {
#// { unitConfig = {OnFailure = "unit-status-mail@${unit}.service";};
# systemd.services = }
# lib.listToAttrs );
# (map ( };
# unit: {
# name = unit;
# value = {
# unitConfig = {OnFailure = "unit-status-mail@${unit}.service";};
# };
# }
# )
# config.mj.base.unitstatus.units);
};
} }

64
modules/base/zfsborg/default.nix
View File

@ -57,34 +57,44 @@ in {
services.borgbackup.jobs = lib.mapAttrs' (mountpoint: attrs: let services.borgbackup.jobs = lib.mapAttrs' (mountpoint: attrs: let
fs = builtins.getAttr mountpoint config.fileSystems; fs = builtins.getAttr mountpoint config.fileSystems;
in in
assert fs.fsType == "zfs"; { assert fs.fsType == "zfs";
name = lib.strings.sanitizeDerivationName mountpoint; assert lib.assertMsg
value = config.mj.base.unitstatus.enable
{ "config.mj.base.unitstatus.enable must be true";
doInit = true; {
repo = config.mj.base.zfsborg.repo; name = lib.strings.sanitizeDerivationName mountpoint;
encryption = { value =
mode = "repokey-blake2"; {
passCommand = "cat ${config.mj.base.zfsborg.passwdPath}"; doInit = true;
repo = config.mj.base.zfsborg.repo;
encryption = {
mode = "repokey-blake2";
passCommand = "cat ${config.mj.base.zfsborg.passwdPath}";
};
paths = attrs.paths;
extraArgs = "--remote-path=borg1";
compression = "auto,lzma";
startAt = attrs.backup_at;
readWritePaths = let p = mountpoint + "/.snapshot-latest"; in [p];
preHook = mountLatest mountpoint fs.device;
postHook = umountLatest mountpoint;
prune.keep = {
within = "1d";
daily = 7;
weekly = 4;
monthly = 3;
};
}
// lib.optionalAttrs (attrs ? patterns) {
patterns = attrs.patterns;
}; };
paths = attrs.paths; })
extraArgs = "--remote-path=borg1";
compression = "auto,lzma";
startAt = attrs.backup_at;
readWritePaths = let p = mountpoint + "/.snapshot-latest"; in [p];
preHook = mountLatest mountpoint fs.device;
postHook = umountLatest mountpoint;
prune.keep = {
within = "1d";
daily = 7;
weekly = 4;
monthly = 3;
};
}
// lib.optionalAttrs (attrs ? patterns) {
patterns = attrs.patterns;
};
})
config.mj.base.zfsborg.mountpoints; config.mj.base.zfsborg.mountpoints;
mj.base.unitstatus.units = let
mounts = config.mj.base.zfsborg.mountpoints;
sanitized = map lib.strings.sanitizeDerivationName (lib.attrNames mounts);
in
map (n: "borgbackup-job-${n}") sanitized;
}; };
} }