diff --git a/hosts/lab/default.nix b/hosts/lab/default.nix index e8192ac..0434586 100644 --- a/hosts/lab/default.nix +++ b/hosts/lab/default.nix @@ -13,6 +13,7 @@ ./tinyauth.nix ./shlink.nix ./guestbook.nix + ./telegram-alerts.nix ]; networking.hostName = "lab"; diff --git a/hosts/lab/telegram-alerts.nix b/hosts/lab/telegram-alerts.nix new file mode 100644 index 0000000..fd3b600 --- /dev/null +++ b/hosts/lab/telegram-alerts.nix @@ -0,0 +1,58 @@ +{ config, pkgs, lib, ... }: +let + chatId = "8669496383"; + host = config.networking.hostName; + + alertScript = pkgs.writeShellScript "telegram-alert" '' + set -u + unit="$1" + token=$(tr -d '\n' < ${config.sops.secrets.telegram-alert-token.path}) + state=$(${pkgs.systemd}/bin/systemctl is-failed "$unit" 2>/dev/null || true) + log=$(${pkgs.systemd}/bin/journalctl -u "$unit" -n 30 --no-pager -o cat 2>/dev/null | tail -c 3500) + text="[${host}] $unit failed (state: $state) + + --- last log --- + $log" + ${pkgs.curl}/bin/curl -fsS --max-time 10 \ + -X POST "https://api.telegram.org/bot$token/sendMessage" \ + --data-urlencode "chat_id=${chatId}" \ + --data-urlencode "text=$text" \ + --data-urlencode "disable_web_page_preview=true" >/dev/null + ''; + + alertedServices = [ + "forgejo" + "caddy" + "postgresql" + "guestbook" + "podman-foundry" + "podman-dokuwiki" + "podman-shlink" + "podman-shlink-web-client" + "podman-uptime-kuma" + "podman-tinyauth" + "site-webhook" + ]; +in +{ + sops.secrets.telegram-alert-token = { + sopsFile = ../../secrets/guestbook.yaml; + key = "telegram_bot_token"; + mode = "0400"; + }; + + systemd.services = lib.mkMerge [ + { + "telegram-alert@" = { + description = "Send Telegram alert for failed unit %i"; + serviceConfig = { + Type = "oneshot"; + ExecStart = "${alertScript} %i"; + }; + }; + } + (lib.genAttrs alertedServices (_: { + unitConfig.OnFailure = [ "telegram-alert@%n.service" ]; + })) + ]; +}