implemented job metrics

README.md

@@ -17,6 +17,15 @@ defaults:
     weekday: "{{ range(6)|list }}"
     hour: "{{ range(1, 22)|list }}"
     minute: "{{ range(5, 50)|list }}"
+
+# Enable metrics for all jobs with node-exporter:
+# * cron_job_last_run_seconds
+# * cron_job_exit_code
+# * cron_job_duration_seconds
+# * cron_job_next_run_seconds
+# * cron_job_last_success_seconds
+# Label "name" is set to the job name (key in jobs dict)
+enable_monitoring: True
 ```
 
 **jobconfig**

defaults/main.yml

@@ -1,5 +1,6 @@
 cron:
   jobs: {}
+  enable_monitoring: False
   defaults:
     user: root
     weekday: '*'

files/monitor-cronjob.py

@@ -0,0 +1,60 @@
+#!/bin/python3
+import time
+import sys
+import re
+import os
+import croniter
+import subprocess
+
+name = sys.argv[1]
+cron_expr = sys.argv[2]
+cmd = sys.argv[3]
+exporterdir = '/var/lib/prometheus/node-exporter'
+
+label_name = name.replace('\\', '\\\\').replace('"', '\\"').replace('\n', '\\n')
+file_name = re.sub('[^a-zA-Z0-9]', '_', name)
+
+start_ts = time.time()
+result = subprocess.run(cmd, shell=True)
+end_ts = time.time()
+
+next_ts = croniter.croniter(cron_expr).get_next()
+
+jobstats = f'''# Generated by monitor-cronjob.py
+
+# HELP cron_job_last_run_seconds Unix timestamp of last run
+# TYPE cron_job_last_run_seconds counter
+cron_job_last_run_seconds{{name="{ label_name }"}} { start_ts }
+
+# HELP cron_job_exit_code Exit code of last run
+# TYPE cron_job_exit_code gauge
+cron_job_exit_code{{name="{ label_name }"}} { result.returncode }
+
+# HELP cron_job_duration_seconds Duration of last run
+# TYPE cron_job_duration_seconds gauge
+cron_job_duration_seconds{{name="{ label_name }"}} { end_ts - start_ts }
+
+# HELP cron_job_next_run_seconds Unix timestamp of next run
+# TYPE cron_job_next_run_seconds counter
+cron_job_next_run_seconds{{name="{ label_name }"}} { next_ts }
+'''
+
+successstats = f'''# Generated by monitor-cronjob.py
+
+# HELP cron_job_last_success_seconds Unix timestamp of last successful run
+# TYPE cron_job_last_success_seconds counter
+cron_job_last_success_seconds{{name="{ label_name }"}} { start_ts }
+'''
+
+tmpsuffix = '.%d'%(os.getpid())
+
+path = os.path.join(exporterdir, f'cron_job_generic_{ file_name }.prom')
+with open(path + tmpsuffix, 'w') as f:
+	f.write(jobstats)
+os.rename(path + tmpsuffix, path)
+
+if result.returncode == 0:
+	path = os.path.join(exporterdir, f'cron_job_success_{ file_name }.prom')
+	with open(path + tmpsuffix, 'w') as f:
+		f.write(successstats)
+	os.rename(path + tmpsuffix, path)

tasks/job.yml

@@ -2,31 +2,38 @@
     job: '{{ {}|combine(cron.defaults, item.value, {"name": item.key}, recursive=True) }}'
     randomseed: "{{ inventory_hostname + item.key }}"
 
-- name: add cron jobs (random_daily)
-  when: job.special_time == "random_daily"
-  cron:
-    name: "{{ job.name }}"
-    job: "{{ job.job }}"
-    user: "{{ job.user }}"
-    hour: "{{ job.random_options.hour | random(seed=(randomseed + 'hour')) }}"
-    minute: "{{ job.random_options.minute | random(seed=(randomseed + 'minute')) }}"
+- when: job.special_time == "random_daily"
+  set_fact:
+    job: '{{ job|combine({"weekday": "*"}) }}'
 
-- name: add cron jobs (random_weekly)
-  when: job.special_time == "random_weekly"
-  cron:
-    name: "{{ job.name }}"
-    job: "{{ job.job }}"
-    user: "{{ job.user }}"
-    weekday: "{{ job.random_options.weekday | random(seed=(randomseed + 'weekday')) }}"
-    hour: "{{ job.random_options.hour | random(seed=(randomseed + 'hour')) }}"
-    minute: "{{ job.random_options.minute | random(seed=(randomseed + 'minute')) }}"
+- when: job.special_time == "random_weekly"
+  set_fact:
+    job: '{{ job|combine({"weekday": job.random_options.weekday | random(seed=(randomseed + "weekday"))}) }}'
 
-- name: add cron jobs (not special)
-  when: not job.special_time
+- when: job.special_time == "random_daily" or job.special_time == "random_weekly"
+  set_fact:
+    job: '{{ job|combine({"hour": job.random_options.hour | random(seed=(randomseed + "hour"))}) }}'
+
+- when: job.special_time == "random_daily" or job.special_time == "random_weekly"
+  set_fact:
+    job: '{{ job|combine({"minute": job.random_options.minute | random(seed=(randomseed + "minute"))}) }}'
+
+- name: add cron job users to node-exporter-textfile group
+  when: cron.enable_monitoring
+  ansible.builtin.user:
+    name: '{{ job.user }}'
+    groups: node-exporter-textfile
+    append: yes
+
+- when: cron.enable_monitoring
+  set_fact:
+    job: '{{ job|combine({"job": "/usr/local/bin/monitor-cronjob.py %s %s %s"%(job.name|quote, ("%s %s * * %s"%(job.minute, job.hour, job.weekday))|quote, job.job|quote)}) }}'
+
+- name: add cron jobs
   cron:
-    name: "{{ job.name }}"
-    job: "{{ job.job }}"
-    user: "{{ job.user }}"
-    weekday: "{{ job.weekday }}"
-    hour: "{{ job.hour }}"
-    minute: "{{ job.minute }}"
+    name: '{{ job.name }}'
+    job: '{{ job.job }}'
+    user: '{{ job.user }}'
+    weekday: '{{ job.weekday }}'
+    hour: '{{ job.hour }}'
+    minute: '{{ job.minute }}'

tasks/main.yml

@@ -1,3 +1,16 @@
+- name: install python3-croniter
+  when: cron.enable_monitoring
+  apt:
+    pkg: python3-croniter
+- name: copy monitor-cronjob.py script
+  when: cron.enable_monitoring
+  ansible.builtin.copy:
+    src: monitor-cronjob.py
+    dest: /usr/local/bin/monitor-cronjob.py
+    mode: 0755
+    owner: root
+    group: root
+
 - include_tasks:
     file: job.yml
   with_dict: "{{ cron.jobs }}"