added support for telegram notifications

This commit is contained in:
nd 2020-06-04 01:25:44 +02:00
parent 0f748dd02a
commit 069df8c553
No known key found for this signature in database
GPG key ID: 21B5CD4DEE3670E9
5 changed files with 107 additions and 2 deletions

View file

@ -1,4 +1,5 @@
prometheus_alertmanager:
telegrambot: False
args:
"web.listen-address": "[::1]:9093"
config:
@ -25,7 +26,7 @@ prometheus_alertmanager:
# resend them.
repeat_interval: 3h
# A default receiver
receiver: mail-default
receiver: default
# All the above attributes are inherited by all child routes and can
# overwritten on each.
# The child route trees.
@ -43,7 +44,7 @@ prometheus_alertmanager:
equal: ['alertname', 'cluster', 'service']
receivers:
- name: "blackhole"
- name: 'mail-default'
- name: 'default'
send_resolved: True
email_configs:
- to: 'root@localhost'

View file

@ -0,0 +1,14 @@
[Unit]
Description=Sends alerts via telegram
After=network.target
[Service]
ExecStart=/usr/local/bin/prometheus-telegram-bot.py PROD
DynamicUser=yes
NoNewPrivileges=True
ProtectSystem=strict
[Install]
WantedBy=multi-user.target

View file

@ -2,3 +2,9 @@
service:
name: prometheus-alertmanager
state: restarted
- name: restart telegram bot
service:
name: prometheus-telegram-bot
enabled: True
daemon_reload: True
state: restarted

View file

@ -16,3 +16,33 @@
mode: 0644
dest: /etc/prometheus/alertmanager.yml
content: "{{ prometheus_alertmanager.config|to_nice_yaml(indent=2) }}"
- name: setup telegram bot
when: prometheus_alertmanager.telegrambot
block:
- name: install dependencies
apt:
pkg:
- python3-flask
- python3-dateutil
- python3-gevent
- python3-pip
# - python3-telegram-bot
- pip:
name: python-telegram-bot
executable: pip3
- name: deploy telegram bot
notify: restart telegram bot
template:
src: telegram-bot.py.j2
dest: /usr/local/bin/prometheus-telegram-bot.py
mode: 0755
owner: root
group: root
- name: Copy systemd service file
notify: restart telegram bot
copy:
src: prometheus-telegram-bot.service
dest: /etc/systemd/system
owner: root
group: root

54
templates/telegram-bot.py.j2 Executable file
View file

@ -0,0 +1,54 @@
#!/usr/bin/env python3
import os
import sys
from dateutil import parser
import telegram, json, logging
from flask import Flask, request
listenPort = 29119
listenIP = '::1'
botToken = '{{ prometheus_alertmanager.telegrambot }}'
app = Flask(__name__)
app.secret_key = os.urandom(128)
bot = telegram.Bot(token=botToken)
@app.route('/<chatID>/alert', methods = ['POST'])
def postAlertmanager(chatID):
try:
content = json.loads(request.get_data())
for alert in content['alerts']:
message = "Status: "+alert['status']+"\n"
if 'name' in alert['labels']:
message += "Instance: "+alert['labels']['instance']+"("+alert['labels']['name']+")\n"
else:
message += "Instance: "+alert['labels']['instance']+"\n"
if 'info' in alert['annotations']:
message += "Info: "+alert['annotations']['info']+"\n"
if 'summary' in alert['annotations']:
message += "Summary: "+alert['annotations']['summary']+"\n"
if 'description' in alert['annotations']:
message += "Description: "+alert['annotations']['description']+"\n"
if alert['status'] == "resolved":
correctDate = parser.parse(alert['endsAt']).strftime('%Y-%m-%d %H:%M:%S')
message += "Resolved: "+correctDate
elif alert['status'] == "firing":
correctDate = parser.parse(alert['startsAt']).strftime('%Y-%m-%d %H:%M:%S')
message += "Started: "+correctDate
bot.sendMessage(chat_id=chatID, text=message)
return "Alert OK", 200
except Exception as error:
bot.sendMessage(chat_id=chatID, text="Error to read json: "+str(error))
app.logger.info("\t%s",error)
return "Alert fail", 200
if __name__ == '__main__':
if len(sys.argv) == 1:
logging.basicConfig(level=logging.INFO)
app.run(host=listenIP, port=listenPort)
else:
from gevent.pywsgi import WSGIServer
http_server = WSGIServer((listenIP, listenPort), app)
http_server.serve_forever()