diff --git a/README.md b/README.md index 527e3d3..45273ea 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,15 @@ defaults: weekday: "{{ range(6)|list }}" hour: "{{ range(1, 22)|list }}" minute: "{{ range(5, 50)|list }}" + +# Enable metrics for all jobs with node-exporter: +# * cron_job_last_run_seconds +# * cron_job_exit_code +# * cron_job_duration_seconds +# * cron_job_next_run_seconds +# * cron_job_last_success_seconds +# Label "name" is set to the job name (key in jobs dict) +enable_monitoring: True ``` **jobconfig** diff --git a/defaults/main.yml b/defaults/main.yml index 48d013f..ad80abb 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -1,5 +1,6 @@ cron: jobs: {} + enable_monitoring: False defaults: user: root weekday: '*' diff --git a/files/monitor-cronjob.py b/files/monitor-cronjob.py new file mode 100755 index 0000000..77661d1 --- /dev/null +++ b/files/monitor-cronjob.py @@ -0,0 +1,60 @@ +#!/bin/python3 +import time +import sys +import re +import os +import croniter +import subprocess + +name = sys.argv[1] +cron_expr = sys.argv[2] +cmd = sys.argv[3] +exporterdir = '/var/lib/prometheus/node-exporter' + +label_name = name.replace('\\', '\\\\').replace('"', '\\"').replace('\n', '\\n') +file_name = re.sub('[^a-zA-Z0-9]', '_', name) + +start_ts = time.time() +result = subprocess.run(cmd, shell=True) +end_ts = time.time() + +next_ts = croniter.croniter(cron_expr).get_next() + +jobstats = f'''# Generated by monitor-cronjob.py + +# HELP cron_job_last_run_seconds Unix timestamp of last run +# TYPE cron_job_last_run_seconds counter +cron_job_last_run_seconds{{name="{ label_name }"}} { start_ts } + +# HELP cron_job_exit_code Exit code of last run +# TYPE cron_job_exit_code gauge +cron_job_exit_code{{name="{ label_name }"}} { result.returncode } + +# HELP cron_job_duration_seconds Duration of last run +# TYPE cron_job_duration_seconds gauge +cron_job_duration_seconds{{name="{ label_name }"}} { end_ts - start_ts } + +# HELP cron_job_next_run_seconds Unix timestamp of next run +# TYPE cron_job_next_run_seconds counter +cron_job_next_run_seconds{{name="{ label_name }"}} { next_ts } +''' + +successstats = f'''# Generated by monitor-cronjob.py + +# HELP cron_job_last_success_seconds Unix timestamp of last successful run +# TYPE cron_job_last_success_seconds counter +cron_job_last_success_seconds{{name="{ label_name }"}} { start_ts } +''' + +tmpsuffix = '.%d'%(os.getpid()) + +path = os.path.join(exporterdir, f'cron_job_generic_{ file_name }.prom') +with open(path + tmpsuffix, 'w') as f: + f.write(jobstats) +os.rename(path + tmpsuffix, path) + +if result.returncode == 0: + path = os.path.join(exporterdir, f'cron_job_success_{ file_name }.prom') + with open(path + tmpsuffix, 'w') as f: + f.write(successstats) + os.rename(path + tmpsuffix, path) diff --git a/tasks/job.yml b/tasks/job.yml index 958542c..6462606 100644 --- a/tasks/job.yml +++ b/tasks/job.yml @@ -2,31 +2,38 @@ job: '{{ {}|combine(cron.defaults, item.value, {"name": item.key}, recursive=True) }}' randomseed: "{{ inventory_hostname + item.key }}" -- name: add cron jobs (random_daily) - when: job.special_time == "random_daily" - cron: - name: "{{ job.name }}" - job: "{{ job.job }}" - user: "{{ job.user }}" - hour: "{{ job.random_options.hour | random(seed=(randomseed + 'hour')) }}" - minute: "{{ job.random_options.minute | random(seed=(randomseed + 'minute')) }}" +- when: job.special_time == "random_daily" + set_fact: + job: '{{ job|combine({"weekday": "*"}) }}' -- name: add cron jobs (random_weekly) - when: job.special_time == "random_weekly" - cron: - name: "{{ job.name }}" - job: "{{ job.job }}" - user: "{{ job.user }}" - weekday: "{{ job.random_options.weekday | random(seed=(randomseed + 'weekday')) }}" - hour: "{{ job.random_options.hour | random(seed=(randomseed + 'hour')) }}" - minute: "{{ job.random_options.minute | random(seed=(randomseed + 'minute')) }}" +- when: job.special_time == "random_weekly" + set_fact: + job: '{{ job|combine({"weekday": job.random_options.weekday | random(seed=(randomseed + "weekday"))}) }}' -- name: add cron jobs (not special) - when: not job.special_time +- when: job.special_time == "random_daily" or job.special_time == "random_weekly" + set_fact: + job: '{{ job|combine({"hour": job.random_options.hour | random(seed=(randomseed + "hour"))}) }}' + +- when: job.special_time == "random_daily" or job.special_time == "random_weekly" + set_fact: + job: '{{ job|combine({"minute": job.random_options.minute | random(seed=(randomseed + "minute"))}) }}' + +- name: add cron job users to node-exporter-textfile group + when: cron.enable_monitoring + ansible.builtin.user: + name: '{{ job.user }}' + groups: node-exporter-textfile + append: yes + +- when: cron.enable_monitoring + set_fact: + job: '{{ job|combine({"job": "/usr/local/bin/monitor-cronjob.py %s %s %s"%(job.name|quote, ("%s %s * * %s"%(job.minute, job.hour, job.weekday))|quote, job.job|quote)}) }}' + +- name: add cron jobs cron: - name: "{{ job.name }}" - job: "{{ job.job }}" - user: "{{ job.user }}" - weekday: "{{ job.weekday }}" - hour: "{{ job.hour }}" - minute: "{{ job.minute }}" + name: '{{ job.name }}' + job: '{{ job.job }}' + user: '{{ job.user }}' + weekday: '{{ job.weekday }}' + hour: '{{ job.hour }}' + minute: '{{ job.minute }}' diff --git a/tasks/main.yml b/tasks/main.yml index c9675ff..d652fd0 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -1,3 +1,16 @@ +- name: install python3-croniter + when: cron.enable_monitoring + apt: + pkg: python3-croniter +- name: copy monitor-cronjob.py script + when: cron.enable_monitoring + ansible.builtin.copy: + src: monitor-cronjob.py + dest: /usr/local/bin/monitor-cronjob.py + mode: 0755 + owner: root + group: root + - include_tasks: file: job.yml with_dict: "{{ cron.jobs }}"