added support for telegram notifications
This commit is contained in:
parent
0f748dd02a
commit
069df8c553
5 changed files with 107 additions and 2 deletions
|
|
@ -1,4 +1,5 @@
|
||||||
prometheus_alertmanager:
|
prometheus_alertmanager:
|
||||||
|
telegrambot: False
|
||||||
args:
|
args:
|
||||||
"web.listen-address": "[::1]:9093"
|
"web.listen-address": "[::1]:9093"
|
||||||
config:
|
config:
|
||||||
|
|
@ -25,7 +26,7 @@ prometheus_alertmanager:
|
||||||
# resend them.
|
# resend them.
|
||||||
repeat_interval: 3h
|
repeat_interval: 3h
|
||||||
# A default receiver
|
# A default receiver
|
||||||
receiver: mail-default
|
receiver: default
|
||||||
# All the above attributes are inherited by all child routes and can
|
# All the above attributes are inherited by all child routes and can
|
||||||
# overwritten on each.
|
# overwritten on each.
|
||||||
# The child route trees.
|
# The child route trees.
|
||||||
|
|
@ -43,7 +44,7 @@ prometheus_alertmanager:
|
||||||
equal: ['alertname', 'cluster', 'service']
|
equal: ['alertname', 'cluster', 'service']
|
||||||
receivers:
|
receivers:
|
||||||
- name: "blackhole"
|
- name: "blackhole"
|
||||||
- name: 'mail-default'
|
- name: 'default'
|
||||||
send_resolved: True
|
send_resolved: True
|
||||||
email_configs:
|
email_configs:
|
||||||
- to: 'root@localhost'
|
- to: 'root@localhost'
|
||||||
|
|
|
||||||
14
files/prometheus-telegram-bot.service
Normal file
14
files/prometheus-telegram-bot.service
Normal file
|
|
@ -0,0 +1,14 @@
|
||||||
|
[Unit]
|
||||||
|
Description=Sends alerts via telegram
|
||||||
|
After=network.target
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
ExecStart=/usr/local/bin/prometheus-telegram-bot.py PROD
|
||||||
|
|
||||||
|
DynamicUser=yes
|
||||||
|
NoNewPrivileges=True
|
||||||
|
ProtectSystem=strict
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=multi-user.target
|
||||||
|
|
||||||
|
|
@ -2,3 +2,9 @@
|
||||||
service:
|
service:
|
||||||
name: prometheus-alertmanager
|
name: prometheus-alertmanager
|
||||||
state: restarted
|
state: restarted
|
||||||
|
- name: restart telegram bot
|
||||||
|
service:
|
||||||
|
name: prometheus-telegram-bot
|
||||||
|
enabled: True
|
||||||
|
daemon_reload: True
|
||||||
|
state: restarted
|
||||||
|
|
|
||||||
|
|
@ -16,3 +16,33 @@
|
||||||
mode: 0644
|
mode: 0644
|
||||||
dest: /etc/prometheus/alertmanager.yml
|
dest: /etc/prometheus/alertmanager.yml
|
||||||
content: "{{ prometheus_alertmanager.config|to_nice_yaml(indent=2) }}"
|
content: "{{ prometheus_alertmanager.config|to_nice_yaml(indent=2) }}"
|
||||||
|
|
||||||
|
- name: setup telegram bot
|
||||||
|
when: prometheus_alertmanager.telegrambot
|
||||||
|
block:
|
||||||
|
- name: install dependencies
|
||||||
|
apt:
|
||||||
|
pkg:
|
||||||
|
- python3-flask
|
||||||
|
- python3-dateutil
|
||||||
|
- python3-gevent
|
||||||
|
- python3-pip
|
||||||
|
# - python3-telegram-bot
|
||||||
|
- pip:
|
||||||
|
name: python-telegram-bot
|
||||||
|
executable: pip3
|
||||||
|
- name: deploy telegram bot
|
||||||
|
notify: restart telegram bot
|
||||||
|
template:
|
||||||
|
src: telegram-bot.py.j2
|
||||||
|
dest: /usr/local/bin/prometheus-telegram-bot.py
|
||||||
|
mode: 0755
|
||||||
|
owner: root
|
||||||
|
group: root
|
||||||
|
- name: Copy systemd service file
|
||||||
|
notify: restart telegram bot
|
||||||
|
copy:
|
||||||
|
src: prometheus-telegram-bot.service
|
||||||
|
dest: /etc/systemd/system
|
||||||
|
owner: root
|
||||||
|
group: root
|
||||||
|
|
|
||||||
54
templates/telegram-bot.py.j2
Executable file
54
templates/telegram-bot.py.j2
Executable file
|
|
@ -0,0 +1,54 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
from dateutil import parser
|
||||||
|
import telegram, json, logging
|
||||||
|
from flask import Flask, request
|
||||||
|
|
||||||
|
listenPort = 29119
|
||||||
|
listenIP = '::1'
|
||||||
|
botToken = '{{ prometheus_alertmanager.telegrambot }}'
|
||||||
|
|
||||||
|
app = Flask(__name__)
|
||||||
|
app.secret_key = os.urandom(128)
|
||||||
|
|
||||||
|
bot = telegram.Bot(token=botToken)
|
||||||
|
|
||||||
|
@app.route('/<chatID>/alert', methods = ['POST'])
|
||||||
|
def postAlertmanager(chatID):
|
||||||
|
|
||||||
|
try:
|
||||||
|
content = json.loads(request.get_data())
|
||||||
|
for alert in content['alerts']:
|
||||||
|
message = "Status: "+alert['status']+"\n"
|
||||||
|
if 'name' in alert['labels']:
|
||||||
|
message += "Instance: "+alert['labels']['instance']+"("+alert['labels']['name']+")\n"
|
||||||
|
else:
|
||||||
|
message += "Instance: "+alert['labels']['instance']+"\n"
|
||||||
|
if 'info' in alert['annotations']:
|
||||||
|
message += "Info: "+alert['annotations']['info']+"\n"
|
||||||
|
if 'summary' in alert['annotations']:
|
||||||
|
message += "Summary: "+alert['annotations']['summary']+"\n"
|
||||||
|
if 'description' in alert['annotations']:
|
||||||
|
message += "Description: "+alert['annotations']['description']+"\n"
|
||||||
|
if alert['status'] == "resolved":
|
||||||
|
correctDate = parser.parse(alert['endsAt']).strftime('%Y-%m-%d %H:%M:%S')
|
||||||
|
message += "Resolved: "+correctDate
|
||||||
|
elif alert['status'] == "firing":
|
||||||
|
correctDate = parser.parse(alert['startsAt']).strftime('%Y-%m-%d %H:%M:%S')
|
||||||
|
message += "Started: "+correctDate
|
||||||
|
bot.sendMessage(chat_id=chatID, text=message)
|
||||||
|
return "Alert OK", 200
|
||||||
|
except Exception as error:
|
||||||
|
bot.sendMessage(chat_id=chatID, text="Error to read json: "+str(error))
|
||||||
|
app.logger.info("\t%s",error)
|
||||||
|
return "Alert fail", 200
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
if len(sys.argv) == 1:
|
||||||
|
logging.basicConfig(level=logging.INFO)
|
||||||
|
app.run(host=listenIP, port=listenPort)
|
||||||
|
else:
|
||||||
|
from gevent.pywsgi import WSGIServer
|
||||||
|
http_server = WSGIServer((listenIP, listenPort), app)
|
||||||
|
http_server.serve_forever()
|
||||||
Loading…
Add table
Add a link
Reference in a new issue