promtail: Add more systemd-journal fields as structured metadata

defaults/main.yml

@@ -41,6 +41,7 @@ prometheus_agent:
               preferred_ip_protocol: ip4
             prober: icmp
       jobs: {}
+
     promtail:
       enable: False
       config:
@@ -52,7 +53,7 @@ prometheus_agent:
           filename: /var/lib/promtail/positions.yaml
         # clients is generated based on prometheus_agent.scrapers
         # scrape_configs is generated based on prometheus_agent.agents.promtail.scrape_jobs
-      # "scrape_jobs" items have the same format as the "scrape_jobs" promtail
+      # "scrape_jobs" items have the same format as the "scrape_configs" promtail
       # config key. However, using a dictionary simplifies extending or changing
       # the default scrape configs. Items with an empty value are ignored.
       # The "job_name" field defaults to the item key.
@@ -62,14 +63,86 @@ prometheus_agent:
             max_age: 12h
             labels:
               job: systemd-journal
+              service_name: other
           relabel_configs:
-            - source_labels: ['__journal__systemd_unit']
+          # User Journal Fields
-              target_label: 'unit'
           - source_labels: ['__journal_priority_keyword']
-              target_label: 'level'
+            target_label: level
+            regex: '(.+)'
+          - source_labels: ['__journal_syslog_facility']
+            target_label: syslog_facility
+            regex: '(.+)'
+          - source_labels: ['__journal_syslog_identifier']
+            target_label: syslog_identifier
+            regex: '(.+)'
+          - source_labels: ['__journal_tid']
+            target_label: tid
+            regex: '(.+)'
+          # Trusted Journal Fields
+          - source_labels: ['__journal__pid']
+            target_label: pid
+            regex: '(.+)'
+          - source_labels: ['__journal__uid']
+            target_label: uid
+            regex: '(.+)'
+          - source_labels: ['__journal__gid']
+            target_label: gid
+            regex: '(.+)'
+          - source_labels: ['__journal__systemd_unit']
+            target_label: systemd_unit
+            regex: '(.+)'
+          - source_labels: ['__journal__systemd_user_unit']
+            target_label: systemd_user_unit
+            regex: '(.+)'
+          - source_labels: ['__journal__boot_id']
+            target_label: boot_id
+            regex: '(.+)'
+          - source_labels: ['__journal__systemd_invocation_id']
+            target_label: systemd_invocation_id
+            regex: '(.+)'
+          - source_labels: ['__journal__transport']
+            target_label: transport
+            regex: '(.+)'
+          - source_labels: ['__journal__stream_id']
+            target_label: stream_id
+            regex: '(.+)'
+          # Kernel Journal Fields
+          - source_labels: ['__journal__kernel_device']
+            target_label: kernel_device
+            regex: '(.+)'
+          - source_labels: ['__journal__kernel_subsystem']
+            target_label: kernel_subsystem
+            regex: '(.+)'
+          # Service name
+          - source_labels: ['__journal__transport', '__journal_syslog_identifier']
+            target_label: service_name
+            regex: 'kernel;(.+)'
+          - source_labels: ['__journal__systemd_unit']
+            target_label: service_name
+            regex: '(.+)'
+          - source_labels: ['__journal__systemd_unit']
+            target_label: service_name
+            regex: 'session-[0-9]+\.scope'
+            replacement: 'session-*.scope'
           pipeline_stages:
           - structured_metadata:
               level:
+              syslog_facility:
+              syslog_identifier:
+              tid:
+              pid:
+              uid:
+              gid:
+              systemd_unit:
+              # service_name is a label
+              systemd_user_unit:
+              boot_id:
+              systemd_invocation_id:
+              transport:
+              stream_id:
+              kernel_device:
+              kernel_subsystem:
+
     snmp:
       enable: False
       mib_path: /usr/share/snmp/mibs

templates/promtail.yml.j2

@@ -10,7 +10,7 @@
 
 {%- if 'scrape_configs' not in config -%}
 {%- set tmp = config.setdefault('scrape_configs', []) -%}
-{%- for key, value in prometheus_agent.agents.promtail.scrape_jobs.items() -%}
+{%- for key, value in prometheus_agent.agents.promtail.scrape_jobs.items() if value -%}
 {%- set tmp = value.setdefault('job_name', key) -%}
 {%- set tmp = config.scrape_configs.append(value) -%}
 {%- endfor -%}