From 5ebe07a48ee44ff42a77a0ee6f9ad48c3866f57a Mon Sep 17 00:00:00 2001 From: Julian Rother Date: Mon, 19 Aug 2024 01:28:29 +0200 Subject: [PATCH] Only reload prometheus/alertmanager if config changes Previously prometheus/alertmanager were unconditionally reloaded every 10 minutes. Reloading alertmanager when it is about to deliver an alert notification seems to make it account successfull notification deliveries as failed, increasing alertmanager_notifications_failed_total. So reloading every 10 minutes caused spurious AlertmanagerNotifications alerts in our setup. Also config updates are now atomic regardless of whether /tmp is a tmpfs. --- files/generate-config.sh | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/files/generate-config.sh b/files/generate-config.sh index 4729a62..540c72d 100644 --- a/files/generate-config.sh +++ b/files/generate-config.sh @@ -1,8 +1,6 @@ #!/bin/bash set -euo pipefail -tmpfile=`mktemp` - ( cat /etc/prometheus/conf.d/*.conf @@ -13,9 +11,14 @@ echo " alertmanagers:" echo "scrape_configs:" cat /etc/prometheus/conf.d/scrape_configs/*.conf -) > $tmpfile +) > /etc/prometheus/prometheus.yml.new -chmod 0644 $tmpfile -mv $tmpfile /etc/prometheus/prometheus.yml -/usr/bin/systemctl reload prometheus -/usr/bin/systemctl reload prometheus-alertmanager || true +chmod 0644 /etc/prometheus/prometheus.yml.new + +if ! diff -q /etc/prometheus/prometheus.yml.new /etc/prometheus/prometheus.yml > /dev/null; then + mv /etc/prometheus/prometheus.yml.new /etc/prometheus/prometheus.yml + /usr/bin/systemctl reload prometheus + /usr/bin/systemctl reload prometheus-alertmanager || true +else + rm -f /etc/prometheus/prometheus.yml.new +fi