From 5045dc7fde13d274b2e07fc246fe6ead21895b4a Mon Sep 17 00:00:00 2001
From: Julian Rother <julian@cccv.de>
Date: Sun, 24 Nov 2024 05:12:55 +0100
Subject: [PATCH] promtail: Add more systemd-journal fields as structured
 metadata

---
 defaults/main.yml         | 87 +++++++++++++++++++++++++++++++++++----
 templates/promtail.yml.j2 |  2 +-
 2 files changed, 81 insertions(+), 8 deletions(-)

diff --git a/defaults/main.yml b/defaults/main.yml
index 8e87fd7..742c355 100644
--- a/defaults/main.yml
+++ b/defaults/main.yml
@@ -41,6 +41,7 @@ prometheus_agent:
               preferred_ip_protocol: ip4
             prober: icmp
       jobs: {}
+
     promtail:
       enable: False
       config:
@@ -52,7 +53,7 @@ prometheus_agent:
           filename: /var/lib/promtail/positions.yaml
         # clients is generated based on prometheus_agent.scrapers
         # scrape_configs is generated based on prometheus_agent.agents.promtail.scrape_jobs
-      # "scrape_jobs" items have the same format as the "scrape_jobs" promtail
+      # "scrape_jobs" items have the same format as the "scrape_configs" promtail
       # config key. However, using a dictionary simplifies extending or changing
       # the default scrape configs. Items with an empty value are ignored.
       # The "job_name" field defaults to the item key.
@@ -62,14 +63,86 @@ prometheus_agent:
             max_age: 12h
             labels:
               job: systemd-journal
+              service_name: other
           relabel_configs:
-            - source_labels: ['__journal__systemd_unit']
-              target_label: 'unit'
-            - source_labels: ['__journal_priority_keyword']
-              target_label: 'level'
+          # User Journal Fields
+          - source_labels: ['__journal_priority_keyword']
+            target_label: level
+            regex: '(.+)'
+          - source_labels: ['__journal_syslog_facility']
+            target_label: syslog_facility
+            regex: '(.+)'
+          - source_labels: ['__journal_syslog_identifier']
+            target_label: syslog_identifier
+            regex: '(.+)'
+          - source_labels: ['__journal_tid']
+            target_label: tid
+            regex: '(.+)'
+          # Trusted Journal Fields
+          - source_labels: ['__journal__pid']
+            target_label: pid
+            regex: '(.+)'
+          - source_labels: ['__journal__uid']
+            target_label: uid
+            regex: '(.+)'
+          - source_labels: ['__journal__gid']
+            target_label: gid
+            regex: '(.+)'
+          - source_labels: ['__journal__systemd_unit']
+            target_label: systemd_unit
+            regex: '(.+)'
+          - source_labels: ['__journal__systemd_user_unit']
+            target_label: systemd_user_unit
+            regex: '(.+)'
+          - source_labels: ['__journal__boot_id']
+            target_label: boot_id
+            regex: '(.+)'
+          - source_labels: ['__journal__systemd_invocation_id']
+            target_label: systemd_invocation_id
+            regex: '(.+)'
+          - source_labels: ['__journal__transport']
+            target_label: transport
+            regex: '(.+)'
+          - source_labels: ['__journal__stream_id']
+            target_label: stream_id
+            regex: '(.+)'
+          # Kernel Journal Fields
+          - source_labels: ['__journal__kernel_device']
+            target_label: kernel_device
+            regex: '(.+)'
+          - source_labels: ['__journal__kernel_subsystem']
+            target_label: kernel_subsystem
+            regex: '(.+)'
+          # Service name
+          - source_labels: ['__journal__transport', '__journal_syslog_identifier']
+            target_label: service_name
+            regex: 'kernel;(.+)'
+          - source_labels: ['__journal__systemd_unit']
+            target_label: service_name
+            regex: '(.+)'
+          - source_labels: ['__journal__systemd_unit']
+            target_label: service_name
+            regex: 'session-[0-9]+\.scope'
+            replacement: 'session-*.scope'
           pipeline_stages:
-            - structured_metadata:
-                level:
+          - structured_metadata:
+              level:
+              syslog_facility:
+              syslog_identifier:
+              tid:
+              pid:
+              uid:
+              gid:
+              systemd_unit:
+              # service_name is a label
+              systemd_user_unit:
+              boot_id:
+              systemd_invocation_id:
+              transport:
+              stream_id:
+              kernel_device:
+              kernel_subsystem:
+
     snmp:
       enable: False
       mib_path: /usr/share/snmp/mibs
diff --git a/templates/promtail.yml.j2 b/templates/promtail.yml.j2
index 03554c1..9b6ef93 100644
--- a/templates/promtail.yml.j2
+++ b/templates/promtail.yml.j2
@@ -10,7 +10,7 @@
 
 {%- if 'scrape_configs' not in config -%}
 {%- set tmp = config.setdefault('scrape_configs', []) -%}
-{%- for key, value in prometheus_agent.agents.promtail.scrape_jobs.items() -%}
+{%- for key, value in prometheus_agent.agents.promtail.scrape_jobs.items() if value -%}
 {%- set tmp = value.setdefault('job_name', key) -%}
 {%- set tmp = config.scrape_configs.append(value) -%}
 {%- endfor -%}