Create custom journald input package

Create a generic package for ingesting journald logs.

packages/journald/_dev/build/build.yml

@@ -0,0 +1,3 @@
+dependencies:
+ ecs:
+ reference: git@1.11

packages/journald/_dev/build/docs/README.md

@@ -0,0 +1,8 @@
+# Journald Input
+
+The journald input integration reads logs from the `journald` system service.
+The journald input reads the log data and the metadata associated with it.
+
+The journald input is available on Linux systems with `systemd` installed.
+
+{{fields "log"}}

packages/journald/_dev/deploy/docker/docker-compose.yml

@@ -0,0 +1,8 @@
+version: '2.3'
+services:
+ journald:
+ image: alpine
+ volumes:
+ - ./sample_logs:/sample_logs:ro
+ - ${SERVICE_LOGS_DIR}:/var/log
+ command: /bin/sh -c "cp /sample_logs/* /var/log/"

packages/journald/changelog.yml

@@ -0,0 +1,6 @@
+# newer versions go on top
+- version: "0.0.1"
+ changes:
+ - description: Initial release of generic journald input package.
+ type: enhancement
+ link: https://github.com/elastic/integrations/pull/0 # FIXME Replace with the real PR link

packages/journald/data_stream/log/_dev/test/pipeline/test-iptables-raw.log

@@ -0,0 +1,21 @@
+<161>Oct 10 07:25:12 Hostname kernel: [wan-lan-default-D]IN=eth0 OUT= MAC=90:10:20:76:8d:20:90:10:65:29:b6:2a:08:00 SRC=158.109.0.1 DST=10.4.0.5 LEN=52 TOS=0x00 PREC=0x00 TTL=63 ID=0 DF PROTO=TCP SPT=38842 DPT=443 WINDOW=2853 RES=0x00 ACK URGP=0
+<6>2021-03-12T14:10:18Z Hostname kernel: [wan-lan-default-D]IN=eth0 OUT= MAC=90:10:20:76:8d:20:90:10:65:29:b6:2a:08:00 SRC=158.109.0.1 DST=10.4.0.5 LEN=52 TOS=0x00 PREC=0x00 TTL=63 ID=0 DF PROTO=TCP SPT=38842 DPT=443 WINDOW=2853 RES=0x00 ACK URGP=0
+2021-03-12T14:10:18Z Hostname kernel: [wan-lan-default-D]IN=eth0 OUT= MAC=90:10:20:76:8d:20:90:10:65:29:b6:2a:08:00 SRC=158.109.0.1 DST=10.4.0.5 LEN=52 TOS=0x00 PREC=0x00 TTL=63 ID=0 DF PROTO=TCP SPT=38842 DPT=443 WINDOW=2853 RES=0x00 ACK URGP=0
+Jan 8 03:37:09 DENY: IN=eth0 OUT= MAC=90:10:28:5f:62:24:90:10:18:5a:89:2a:08:00 SRC=192.0.2.71 DST=192.0.2.83 LEN=88 TOS=0x00 PREC=0x00 TTL=118 ID=21684 PROTO=ICMP TYPE=3 CODE=3 [SRC=192.0.2.83 DST=192.168.173.191 LEN=60 TOS=0x00 PREC=0x00 TTL=54 ID=0 DF PROTO=UDP SPT=21458 DPT=62936 LEN=40 ]
+Jan 8 03:37:09 example-host kernel: iptables DROP_INPUT: IN=eth0 OUT= MAC=90:10:35:5a:1e:3a:90:10:9e:ec:2c:71:08:00 SRC=203.0.113.36 DST=172.16.54.114 LEN=52 TOS=0x00 PREC=0x00 TTL=115 ID=15743 DF PROTO=TCP SPT=17805 DPT=445 WINDOW=8192 RES=0x00 SYN URGP=0 
+Jan 8 03:37:57 example-host kernel: iptables DROP_INPUT: IN=eth0 OUT= MAC=90:10:35:5a:1e:3a:90:10:76:e0:e2:d5:08:00 SRC=198.51.100.198 DST=172.16.54.114 LEN=40 TOS=0x00 PREC=0x00 TTL=243 ID=17703 PROTO=TCP SPT=47091 DPT=1433 WINDOW=1024 RES=0x00 SYN URGP=0 
+Jan 8 03:38:45 example-host kernel: iptables DROP_INPUT: IN=eth0 OUT= MAC=90:10:35:5a:1e:3a:90:10:9e:ec:2c:71:08:00 SRC=203.0.113.201 DST=172.16.54.114 LEN=52 TOS=0x00 PREC=0x00 TTL=115 ID=19619 DF PROTO=TCP SPT=59319 DPT=445 WINDOW=8192 RES=0x00 SYN URGP=0 
+Jan 8 03:39:25 example-host kernel: iptables DROP_INPUT: IN=eth0 OUT= MAC=90:10:35:5a:1e:3a:90:10:9e:ec:2c:71:08:00 SRC=203.0.113.246 DST=172.16.54.114 LEN=40 TOS=0x00 PREC=0x00 TTL=240 ID=4255 DF PROTO=TCP SPT=44181 DPT=80 WINDOW=14600 RES=0x00 SYN URGP=0 
+Jan 8 03:40:21 example-host kernel: iptables DROP_INPUT: IN=eth0 OUT= MAC=90:10:35:5a:1e:3a:90:10:76:e0:e2:d5:08:00 SRC=203.0.113.208 DST=172.16.54.114 LEN=52 TOS=0x00 PREC=0x00 TTL=110 ID=27150 DF PROTO=TCP SPT=64358 DPT=445 WINDOW=8192 RES=0x00 SYN URGP=0 
+Jan 8 03:40:25 example-host kernel: iptables DROP_INPUT: IN=eth0 OUT= MAC=90:10:35:5a:1e:3a:90:10:9e:ec:2c:71:08:00 SRC=198.51.100.160 DST=172.16.54.114 LEN=40 TOS=0x00 PREC=0x00 TTL=242 ID=7264 PROTO=TCP SPT=58830 DPT=445 WINDOW=1024 RES=0x00 SYN URGP=0 
+Jan 8 03:41:17 example-host kernel: iptables DROP_INPUT: IN=eth0 OUT= MAC=90:10:35:5a:1e:3a:90:10:76:e0:e2:d5:08:00 SRC=198.51.100.115 DST=172.16.54.114 LEN=52 TOS=0x00 PREC=0x00 TTL=117 ID=6101 DF PROTO=TCP SPT=51985 DPT=445 WINDOW=8192 RES=0x00 SYN URGP=0 
+Jan 8 03:41:23 example-host kernel: iptables DROP_INPUT: IN=eth0 OUT= MAC=90:10:35:5a:1e:3a:90:10:76:e0:e2:d5:08:00 SRC=198.51.100.167 DST=172.16.54.114 LEN=52 TOS=0x00 PREC=0x00 TTL=45 ID=6319 DF PROTO=TCP SPT=4099 DPT=445 WINDOW=8192 RES=0x00 SYN URGP=0 
+Jan 8 03:43:18 example-host kernel: iptables DROP_INPUT: IN=eth0 OUT= MAC=90:10:35:5a:1e:3a:90:10:9e:ec:2c:71:08:00 SRC=198.51.100.19 DST=172.16.54.114 LEN=40 TOS=0x00 PREC=0x00 TTL=245 ID=48624 PROTO=TCP SPT=59287 DPT=139 WINDOW=1024 RES=0x00 SYN URGP=0 
+Jan 8 03:43:42 example-host kernel: iptables DROP_INPUT: IN=eth0 OUT= MAC=90:10:35:5a:1e:3a:90:10:76:e0:e2:d5:08:00:45:00:00:00:00 SRC=198.51.100.68 DST=172.16.54.114 LEN=40 TOS=0x00 PREC=0x00 TTL=250 ID=54321 PROTO=TCP SPT=53296 DPT=8088 WINDOW=65535 RES=0x00 SYN URGP=0 
+Jan 22 09:05:05 ubuntu-bionic kernel: [16571.459614] IN= OUT=lo SRC=2001:0db8:0000:0000:0000:0000:0000:0001 DST=2001:0db8:0000:0000:0000:0000:0000:0002 LEN=104 TC=0 HOPLIMIT=64 FLOWLBL=868225 PROTO=ICMPv6 TYPE=128 CODE=0 ID=3427 SEQ=1 
+Jan 22 10:52:34 ubuntu-bionic kernel: [ 307.757925] IN= OUT=enp0s3 MAC=90:10:12:34:56:78:90:10:aa:bb:cc:dd:86:dd:ff:ff SRC=fe80:0000:0000:0000:0084:88ff:feae:790a DST=ff02:0000:0000:0000:0000:0000:0000:0016 LEN=96 TC=0 HOPLIMIT=1 FLOWLBL=0 PROTO=ICMPv6 TYPE=143 CODE=0 MARK=0xd4
+Jan 5 20:17:05 MainFirewall kernel: [LAN_LOCAL-default-A]IN=eth0.90 OUT= MAC=90:10:92:6e:ea:a7:90:10:73:ba:d6:77:08:00:45:fc:02:1c SRC=192.168.48.137 DST=255.55.174.225 LEN=540 TOS=0x1C PREC=0xE0 TTL=64 ID=27223 PROTO=UDP SPT=48689 DPT=48689 LEN=520 
+Jan 5 20:17:01 MainFirewall kernel: [WAN_OUT-2000-A]IN=eth0 OUT=eth2 MAC=90:10:20:76:8d:20:90:10:24:67:f4:89:08:00 SRC=192.168.134.158 DST=192.0.2.25 LEN=265 TOS=0x00 PREC=0x00 TTL=63 ID=51768 DF PROTO=TCP SPT=43189 DPT=443 WINDOW=159 RES=0x00 ACK PSH URGP=0 
+Jan 5 20:17:01 MainFirewall kernel: [source-dest-default-D]IN=eth0 OUT=eth2 MAC=90:10:20:76:8d:20:90:10:65:29:b6:2a:08:00 SRC=192.168.110.116 DST=192.0.2.25 LEN=52 TOS=0x00 PREC=0x00 TTL=63 ID=0 DF PROTO=TCP SPT=50093 DPT=1443 WINDOW=2857 RES=0x00 ACK URGP=0 
+Jan 5 20:17:01 MainFirewall kernel: [WAN_OUT-2000-A]IN=eth0 OUT=eth2 MAC=90:10:20:76:8d:20:90:10:65:29:b6:2a:08:00 SRC=192.168.110.116 DST=192.0.2.25 LEN=52 TOS=0x00 PREC=0x00 TTL=63 ID=0 DF PROTO=TCP SPT=50093 DPT=1443 WINDOW=2853 RES=0x00 ACK URGP=0 
+Jan 5 20:17:01 MainFirewall kernel: [WAN_OUT-2000-A]IN=eth0 OUT=eth2 MAC=90:10:20:76:8d:20:90:10:65:29:b6:2a:08:00 SRC=192.168.110.116 DST=192.0.2.25 LEN=52 TOS=0x00 PREC=0x00 TTL=63 ID=0 DF PROTO=TCP SPT=50093 DPT=1443 WINDOW=2850 RES=0x00 ACK URGP=0 

packages/journald/data_stream/log/_dev/test/pipeline/test-iptables-raw.log-config.yml

@@ -0,0 +1,6 @@
+dynamic_fields:
+ event.ingested: "^.*$"
+ "@timestamp": "^[0-9]{4}(-[0-9]{2}){2}T[0-9]{2}(:[0-9]{2}){2}\\.[0-9]{3}Z$"
+fields:
+ tags:
+ - preserve_original_event