add alert for systemd services

defaults/main.yml

@@ -188,6 +188,18 @@ prometheus_agent:
       annotations:
         title: '{%raw%}{{ $labels.instance }}: TempCritical{%endraw%}'
         description: "Too hot!"
+    SystemdServices:
+      group: nodeexporter
+      enabled: True
+      alert: systemd
+      expr: |
+        node_systemd_units{job="node", instance=~"{{ inventory_hostname }}", state="failed"} > 0
+      for: 5m
+      labels:
+        severity: critical
+      annotations:
+        title: '{%raw%}{{ $labels.instance }}: Not all systemd services are running{%endraw%}'
+        description: "Service not running"
 
   tls:
     mode: stunnel