Addon: Collect logs to Kafka topic

addon-logs/logs-kube-kafka.yml

@@ -0,0 +1,46 @@
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+  name: logs-kafka
+  namespace: kube-system
+spec:
+  template:
+    metadata:
+      labels:
+        k8s-app: logs-kafka
+        version: v1
+        kubernetes.io/cluster-service: "true"
+    spec:
+      containers:
+      - name: kafkacat
+        image: solsson/kafkacat@sha256:ebebf47061300b14a4b4c2e1e4303ab29f65e4b95d34af1b14bb8f7ec6da7cef
+        command:
+        - sh
+        - -ec
+        - >
+          cd /var/log/containers/;
+          tail
+          -n 0
+          --follow=name
+          *.log
+          |
+          kafkacat
+          -b kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092
+          -t ops-kube-logs-raw-001
+          -P
+          #-z snappy
+        resources:
+          limits:
+            memory: 100Mi
+          requests:
+            cpu: 100m
+            memory: 100Mi
+        volumeMounts:
+        - name: varlog
+          mountPath: /var/log
+          readOnly: true
+      terminationGracePeriodSeconds: 10
+      volumes:
+      - name: varlog
+        hostPath:
+          path: /var/log

addon-logs/topic-ops-kube-logs-raw.yml

@@ -0,0 +1,32 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: topic-ops-kube-logs-raw
+  namespace: kafka
+spec:
+  template:
+    metadata:
+      labels:
+        app: topic-create
+        topic-id: ops-kube-logs-raw
+        topic-gen: "001"
+    spec:
+      containers:
+      - name: kafka
+        image: solsson/kafka:1.0.0@sha256:17fdf1637426f45c93c65826670542e36b9f3394ede1cb61885c6a4befa8f72d
+        command:
+        - ./bin/kafka-topics.sh
+        - --zookeeper
+        - zookeeper:2181
+        - --create
+        - --if-not-exists
+        - --topic
+        - ops-kube-logs-raw-001
+        - --partitions
+        - "1"
+        - --replication-factor
+        - "1"
+        - --config
+        # this might be eight days
+        - retention.ms=69125000
+      restartPolicy: Never