updated integration of agent

defaults/main.yml

@@ -1,17 +1,11 @@
 prometheus:
+  args:
+    "web.listen-address": "[::1]:9090"
   config:
     global:
       scrape_interval:     15s
       evaluation_interval: 15s
       external_labels:
-          monitor: 'example'
-    alerting:
-      alertmanagers:
-      - static_configs:
-        - targets: ['localhost:9093']
+        scraper: "{{ inventory_hostname }}"
     rule_files:
-    scrape_configs:
-      - job_name: 'prometheus'
-        static_configs:
-          - targets: ['localhost:9090']
-
+      - /etc/prometheus/conf.d/rule_files/*.conf

files/generate-config.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+set -euo pipefail
+
+tmpfile=`mktemp`
+
+(
+
+cat /etc/prometheus/conf.d/*.conf
+echo "alerting:"
+echo "  alertmanagers:"
+(cat /etc/prometheus/conf.d/alerting/*.conf 2> /dev/null | sed "s/^/  /") || echo ""
+
+echo "scrape_configs:"
+cat /etc/prometheus/conf.d/scrape_configs/*.conf
+
+) > $tmpfile
+
+chmod 0644 $tmpfile
+mv $tmpfile /etc/prometheus/prometheus.yml
+/usr/bin/systemctl reload prometheus

handlers/main.yml

@@ -0,0 +1,7 @@
+- name: regenerate prometheus config
+  command: /usr/local/sbin/prometheus-generate-config.sh
+
+- name: restart prometheus
+  service:
+    name: prometheus
+    state: restarted

tasks/main.yml

@@ -8,13 +8,20 @@
     groups: ssl-cert
     append: yes
 
+- name: wrtie prometheus config
+  notify: restart prometheus
+  template:
+    src: prometheus.j2
+    dest: /etc/default/prometheus
+
 - name: wrtie prometheus (base) config
+  notify: regenerate prometheus config
   copy:
     owner: root
     group: root
     mode: 0644
-    dest: /etc/prometheus/conf.d/prometheus.yml
-    content: "{{ prometheus.config|to_nice_yaml }}"
+    dest: /etc/prometheus/conf.d/prometheus.conf
+    content: "{{ prometheus.config|to_nice_yaml(indent=2) }}"
 
 - name: create ssl certificate folder
   file:
@@ -34,9 +41,26 @@
   loop:
     - /etc/prometheus/conf.d
     - /etc/prometheus/conf.d/scrape_configs
+    - /etc/prometheus/conf.d/rule_files
+    - /etc/prometheus/conf.d/alerting
+
+- name: add self scrape job
+  notify: regenerate prometheus config
+  template:
+    src: scrape_self.conf.j2
+    dest: /etc/prometheus/conf.d/scrape_configs/selfscrape.conf
+
+- name: add config generating script
+  notify: regenerate prometheus config
+  copy:
+    dest: /usr/local/sbin/prometheus-generate-config.sh
+    src: generate-config.sh
+    mode: 0755
+    owner: root
+    group: root
 
 - name: add cronjob to regenerate config
   cron:
     name: regenerate prometheus config
     minute: 3,18,33,48
-    job: cat /etc/prometheus/conf.d/* /etc/prometheus/conf.d/scrape_configs/* 2> /dev/null > /etc/prometheus/prometheus.yml; /usr/sbin/service prometheus reload
+    job: /usr/local/sbin/prometheus-generate-config.sh

templates/prometheus.j2

@@ -0,0 +1,57 @@
+# Set the command-line arguments to pass to the server.
+ARGS="{% for i in prometheus.args %} --{{ i }}{% if prometheus.args[i] and prometheus.args[i] != {} %}='{{ prometheus.args[i] }}'{% endif %} {% endfor %}"
+
+# Prometheus supports the following options:
+#  --config.file="/etc/prometheus/prometheus.yml"
+#                             Prometheus configuration file path.
+#  --web.listen-address="0.0.0.0:9090"
+#                             Address to listen on for UI, API, and telemetry.
+#  --web.read-timeout=5m      Maximum duration before timing out read of the
+#                             request, and closing idle connections.
+#  --web.max-connections=512  Maximum number of simultaneous connections.
+#  --web.external-url=<URL>   The URL under which Prometheus is externally
+#                             reachable (for example, if Prometheus is served
+#                             via a reverse proxy). Used for generating
+#                             relative and absolute links back to Prometheus
+#                             itself. If the URL has a path portion, it will
+#                             be used to prefix all HTTP endpoints served by
+#                             Prometheus. If omitted, relevant URL components
+#                             will be derived automatically.
+#  --web.route-prefix=<path>  Prefix for the internal routes of web endpoints.
+#                             Defaults to path of --web.external-url.
+#  --web.local-assets="/usr/share/prometheus/web/"
+#                             Path to static asset/templates directory.
+#  --web.user-assets=<path>   Path to static asset directory, available at
+#                             /user.
+#  --web.enable-lifecycle     Enable shutdown and reload via HTTP request.
+#  --web.enable-admin-api     Enables API endpoints for admin control actions.
+#  --web.console.templates="/etc/prometheus/consoles"
+#                             Path to the console template directory,
+#                             available at /consoles.
+#  --web.console.libraries="/etc/prometheus/console_libraries"
+#                             Path to the console library directory.
+#  --storage.tsdb.path="/var/lib/prometheus/metrics2/"
+#                             Base path for metrics storage.
+#  --storage.tsdb.min-block-duration=2h
+#                             Minimum duration of a data block before being
+#                             persisted.
+#  --storage.tsdb.max-block-duration=<duration>
+#                             Maximum duration compacted blocks may span.
+#                             (Defaults to 10% of the retention period)
+#  --storage.tsdb.retention=15d
+#                             How long to retain samples in the storage.
+#  --storage.tsdb.use-lockfile
+#                             Create a lockfile in data directory.
+#  --alertmanager.notification-queue-capacity=10000
+#                             The capacity of the queue for pending alert
+#                             manager notifications.
+#  --alertmanager.timeout=10s
+#                             Timeout for sending alerts to Alertmanager.
+#  --query.lookback-delta=5m  The delta difference allowed for retrieving
+#                             metrics during expression evaluations.
+#  --query.timeout=2m         Maximum time a query may take before being
+#                             aborted.
+#  --query.max-concurrency=20
+#                             Maximum number of queries executed concurrently.
+#  --log.level=info           Only log messages with the given severity or
+#                             above. One of: [debug, info, warn, error]

templates/scrape_self.conf.j2

@@ -0,0 +1,6 @@
+- job_name: 'prometheus'
+  static_configs:
+  - targets: ['localhost:9090']
+  relabel_configs:
+  - replacement: '{{ inventory_hostname }}'
+    target_label: instance