refactor rocketchat template and alert handling
This commit is contained in:
psy
parent
05063e0077
commit
96e6bc43d8
1 changed files with 144 additions and 54 deletions
|
|
@ -7,12 +7,98 @@ from flask import Flask, request
|
||||||
from rocketchat_API.rocketchat import RocketChat
|
from rocketchat_API.rocketchat import RocketChat
|
||||||
from requests import sessions
|
from requests import sessions
|
||||||
|
|
||||||
|
|
||||||
listenPort = 29120
|
listenPort = 29120
|
||||||
listenIP = '::1'
|
listenIP = '::1'
|
||||||
botUser = '{{ prometheus_alertmanager.rocketchatbot.user }}'
|
botUser = '{{ prometheus_alertmanager.rocketchatbot.user }}'
|
||||||
botPass = '{{ prometheus_alertmanager.rocketchatbot.pass }}'
|
botPass = '{{ prometheus_alertmanager.rocketchatbot.pass }}'
|
||||||
botChatURL ='{{ prometheus_alertmanager.rocketchatbot.url }}'
|
botChatURL = '{{ prometheus_alertmanager.rocketchatbot.url }}'
|
||||||
|
|
||||||
|
# ignore fields while printing labels
|
||||||
|
label_ignore_fields = ['ansible_group_ungrouped']
|
||||||
|
|
||||||
|
icons = {
|
||||||
|
'firing': ':warning:',
|
||||||
|
'resolved': ':white_check_mark:'
|
||||||
|
}
|
||||||
|
|
||||||
|
'''
|
||||||
|
Example attachment:
|
||||||
|
|
||||||
|
message.attachment = [{
|
||||||
|
"color": "#ff0000",
|
||||||
|
"text": "Yay for gruggy!",
|
||||||
|
"ts": "2016-12-09T16:53:06.761Z",
|
||||||
|
"thumb_url": "data:image/gif;base64,...", # add thumb image as base64 if needed
|
||||||
|
"message_link": "https://google.com",
|
||||||
|
"collapsed": false,
|
||||||
|
"author_name": "Bradley Hilton",
|
||||||
|
"author_link": "https://rocket.chat/",
|
||||||
|
"author_icon": "https://avatars.githubusercontent.com/u/850391?v=3",
|
||||||
|
"title": "Attachment Example",
|
||||||
|
"title_link": "https://youtube.com",
|
||||||
|
"title_link_download": true,
|
||||||
|
"image_url": "http://res.guggy.com/logo_128.png",
|
||||||
|
"audio_url": "http://www.w3schools.com/tags/horse.mp3",
|
||||||
|
"video_url": "http://www.w3schools.com/tags/movie.mp4",
|
||||||
|
"fields": [{
|
||||||
|
"short": true,
|
||||||
|
"title": "Test",
|
||||||
|
"value": "Testing out something or other"
|
||||||
|
},{
|
||||||
|
"short": true,
|
||||||
|
"title": "Another Test",
|
||||||
|
"value": "[Link](https://google.com/) something and this and that."
|
||||||
|
}]
|
||||||
|
}]
|
||||||
|
|
||||||
|
|
||||||
|
Example alert json:
|
||||||
|
|
||||||
|
[{
|
||||||
|
"receiver": "default",
|
||||||
|
"status": "resolved",
|
||||||
|
"alerts": [{
|
||||||
|
"status": "resolved",
|
||||||
|
"labels": {
|
||||||
|
"alertname": "ProbeFailed",
|
||||||
|
"ansible_group_ungrouped": "1",
|
||||||
|
"instance": "monitoring.cccv.de",
|
||||||
|
"job": "blackbox",
|
||||||
|
"module": "http_2xx",
|
||||||
|
"scraper": "monitoring.cccv.de",
|
||||||
|
"severity": "critical",
|
||||||
|
"target": "https://shells.darmstadt.ccc.de/~psy/alerttest.html"
|
||||||
|
},
|
||||||
|
"annotations": {
|
||||||
|
"description": "monitoring.cccv.de probe for https://shells.darmstadt.ccc.de/~psy/alerttest.html failed",
|
||||||
|
"title": "monitoring.cccv.de: target https://shells.darmstadt.ccc.de/~psy/alerttest.html failed"
|
||||||
|
},
|
||||||
|
"startsAt": "2021-01-04T15:36:30.85487796Z",
|
||||||
|
"endsAt": "2021-01-04T17:18:30.85487796Z",
|
||||||
|
"generatorURL": "http://monitoring:29090/graph?g0.expr=probe_success%7Binstance%3D%22monitoring.cccv.de%22%2Cjob%3D%22blackbox%22%7D+%3D%3D+0+or+absent%28probe_success%7Binstance%3D%22monitoring.cccv.de%22%2Cjob%3D%22blackbox%22%7D%29&g0.tab=1"
|
||||||
|
}],
|
||||||
|
"groupLabels": {
|
||||||
|
"alertname": "ProbeFailed"
|
||||||
|
},
|
||||||
|
"commonLabels": {
|
||||||
|
"alertname": "ProbeFailed",
|
||||||
|
"ansible_group_ungrouped": "1",
|
||||||
|
"instance": "monitoring.cccv.de",
|
||||||
|
"job": "blackbox",
|
||||||
|
"module": "http_2xx",
|
||||||
|
"scraper": "monitoring.cccv.de",
|
||||||
|
"severity": "critical",
|
||||||
|
"target": "https://shells.darmstadt.ccc.de/~psy/alerttest.html"
|
||||||
|
},
|
||||||
|
"commonAnnotations": {
|
||||||
|
"description": "monitoring.cccv.de probe for https://shells.darmstadt.ccc.de/~psy/alerttest.html failed",
|
||||||
|
"title": "monitoring.cccv.de: target https://shells.darmstadt.ccc.de/~psy/alerttest.html failed"
|
||||||
|
},
|
||||||
|
"externalURL": "http://monitoring:29093",
|
||||||
|
"version": "4",
|
||||||
|
"groupKey": "{}:{alertname=\"ProbeFailed\"}"
|
||||||
|
}]
|
||||||
|
'''
|
||||||
|
|
||||||
app = Flask(__name__)
|
app = Flask(__name__)
|
||||||
app.secret_key = os.urandom(128)
|
app.secret_key = os.urandom(128)
|
||||||
|
|
@ -20,56 +106,59 @@ app.secret_key = os.urandom(128)
|
||||||
api_session = sessions.Session()
|
api_session = sessions.Session()
|
||||||
api = RocketChat(botUser, botPass, server_url=botChatURL, session=api_session)
|
api = RocketChat(botUser, botPass, server_url=botChatURL, session=api_session)
|
||||||
|
|
||||||
@app.route('/<chatName>/alert', methods = ['POST'])
|
|
||||||
|
@app.route('/<chatName>/alert', methods=['POST'])
|
||||||
def postAlertmanager(chatName):
|
def postAlertmanager(chatName):
|
||||||
try:
|
try:
|
||||||
content = json.loads(request.get_data())
|
content = json.loads(request.get_data())
|
||||||
|
except Exception as error:
|
||||||
|
api.chat_post_message("Error to read json: " + str(error), channel=chatName, alias='Alertmanager')
|
||||||
|
app.logger.info("\t%s", error)
|
||||||
|
return "Alert fail", 200
|
||||||
|
|
||||||
res = True
|
success = True
|
||||||
|
|
||||||
for alert in content['alerts']:
|
for alert in content['alerts']:
|
||||||
message = ""
|
try:
|
||||||
if alert['status'] == "firing":
|
send_alert_message(chatName, alert)
|
||||||
message += ":warning: "
|
except Exception as error:
|
||||||
else:
|
success = False
|
||||||
message += ":white_check_mark: "
|
app.logger.info("\t%s", error)
|
||||||
message += " [**"+alert['status']+"**] "
|
api.chat_post_message("Error to post alert: " + str(error), channel=chatName, alias='Alertmanager')
|
||||||
if 'description' in alert['annotations']:
|
|
||||||
message += alert['annotations']['description']
|
|
||||||
if alert['status'] == "firing":
|
|
||||||
message += " @here"
|
|
||||||
message += "\n"
|
|
||||||
if 'name' in alert['labels']:
|
|
||||||
message += "Instance: "+alert['labels'].get('instance', 'unknown')+"("+alert['labels']['name']+")\n"
|
|
||||||
else:
|
|
||||||
message += "Instance: "+alert['labels'].get('instance', 'unknown')+"\n"
|
|
||||||
if 'info' in alert['annotations']:
|
|
||||||
message += "Info: "+alert['annotations']['info']+"\n"
|
|
||||||
if 'summary' in alert['annotations']:
|
|
||||||
message += "Summary: "+alert['annotations']['summary']+"\n"
|
|
||||||
if alert['status'] == "resolved":
|
|
||||||
correctDate = parser.parse(alert['endsAt']).strftime('%Y-%m-%d %H:%M:%S')
|
|
||||||
message += "Resolved: "+correctDate+"\n"
|
|
||||||
elif alert['status'] == "firing":
|
|
||||||
correctDate = parser.parse(alert['startsAt']).strftime('%Y-%m-%d %H:%M:%S')
|
|
||||||
message += "Started: "+correctDate+"\n"
|
|
||||||
message += "labels: \n"
|
|
||||||
message += "```\n"
|
|
||||||
labels = ""
|
|
||||||
for l in alert['labels']:
|
|
||||||
labels += l + " : "+alert['labels'][l] + "\n"
|
|
||||||
message += labels + "```\n"
|
|
||||||
|
|
||||||
res &= api.chat_post_message(message, channel=chatName, alias='Alertmanager').ok
|
if success:
|
||||||
|
|
||||||
if res:
|
|
||||||
return "Alert OK", 200
|
return "Alert OK", 200
|
||||||
else:
|
else:
|
||||||
return "Alert fail", 200
|
return "Alert fail", 200
|
||||||
except Exception as error:
|
|
||||||
api.chat_post_message("Error to read json: "+str(error), channel=chatName, alias='Alertmanager')
|
|
||||||
app.logger.info("\t%s",error)
|
def send_alert_message(channel, alert):
|
||||||
return "Alert fail", 200
|
message = "[**{state}**] [{instance}] {here}".format(state=alert['status'],
|
||||||
|
instance=alert['labels'].get('instance', 'unknown'),
|
||||||
|
here='@here' if alert['status'] == 'firing' else '')
|
||||||
|
attach = []
|
||||||
|
|
||||||
|
# attach details if alert is firing
|
||||||
|
if alert['status'] == 'firing':
|
||||||
|
attach.append({
|
||||||
|
"color": "#ff0000",
|
||||||
|
"text": "\n".join(["**{}**: {}".format(key, value) for key, value in alert['annotations'].items() if
|
||||||
|
key not in ['title']]),
|
||||||
|
"ts": parser.parse(alert['startsAt'] if alert['status'] == 'firing' else alert['endsAt']).isoformat(),
|
||||||
|
"message_link": alert.get('generatorURL', '#'), # link for timestamp, mandatory to display timestamp
|
||||||
|
"collapsed": True, # collapse details by default
|
||||||
|
"author_name": alert['labels'].get('instance', 'unknown'),
|
||||||
|
"title": alert['annotations']['title'],
|
||||||
|
"fields": [{"short": True, "title": key, "value": value} for key, value in alert['labels'].items() if
|
||||||
|
key not in label_ignore_fields]
|
||||||
|
})
|
||||||
|
|
||||||
|
res = api.chat_post_message(message, attachments=attach, channel=channel,
|
||||||
|
alias=alert['labels'].get('alertname', 'unknown alert'),
|
||||||
|
emoji=icons.get(alert['status']))
|
||||||
|
|
||||||
|
if not res.ok:
|
||||||
|
raise Exception(res.content)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
if len(sys.argv) == 1:
|
if len(sys.argv) == 1:
|
||||||
|
|
@ -77,5 +166,6 @@ if __name__ == '__main__':
|
||||||
app.run(host=listenIP, port=listenPort)
|
app.run(host=listenIP, port=listenPort)
|
||||||
else:
|
else:
|
||||||
from gevent.pywsgi import WSGIServer
|
from gevent.pywsgi import WSGIServer
|
||||||
|
|
||||||
http_server = WSGIServer((listenIP, listenPort), app)
|
http_server = WSGIServer((listenIP, listenPort), app)
|
||||||
http_server.serve_forever()
|
http_server.serve_forever()
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue