From 2fe4e2435dfe8fbf36e69cb3a7a2edfaf6477cd9 Mon Sep 17 00:00:00 2001
From: nd <git@notandy.de>
Date: Thu, 6 Aug 2020 09:21:09 +0200
Subject: [PATCH] add alert for systemd services

---
 defaults/main.yml | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/defaults/main.yml b/defaults/main.yml
index 516268a..ebe7017 100644
--- a/defaults/main.yml
+++ b/defaults/main.yml
@@ -188,6 +188,18 @@ prometheus_agent:
       annotations:
         title: '{%raw%}{{ $labels.instance }}: TempCritical{%endraw%}'
         description: "Too hot!"
+    SystemdServices:
+      group: nodeexporter
+      enabled: True
+      alert: systemd
+      expr: |
+        node_systemd_units{job="node", instance=~"{{ inventory_hostname }}", state="failed"} > 0
+      for: 5m
+      labels:
+        severity: critical
+      annotations:
+        title: '{%raw%}{{ $labels.instance }}: Not all systemd services are running{%endraw%}'
+        description: "Service not running"
 
   tls:
     mode: stunnel