feat(elastic): logstash01 service

compose.yml

@@ -158,6 +158,31 @@ services:
       - KIBANA_HOSTS=http://kibana:5601
       - LOGSTASH_HOSTS=http://logstash01:9600
 
+  logstash01:
+    depends_on:
+      es01:
+        condition: service_healthy
+      kibana:
+        condition: service_healthy
+    image: caltrans/pems:logstash01
+    build:
+      context: .
+      dockerfile: ./elasticsearch/logstash01/Dockerfile
+      args:
+        - ELASTIC_STACK_VERSION=${ELASTIC_STACK_VERSION}
+    labels:
+      co.elastic.logs/module: logstash
+    user: root
+    volumes:
+      - certs:/usr/share/logstash/certs
+      - logstashdata01:/usr/share/logstash/data
+      - "./elasticsearch/logstash01/ingest_data/:/usr/share/logstash/ingest_data/"
+    environment:
+      - xpack.monitoring.enabled=false
+      - ELASTIC_USER=elastic
+      - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
+      - ELASTIC_HOSTS=https://es01:9200
+
 volumes:
   certs:
     driver: local
@@ -169,3 +194,5 @@ volumes:
     driver: local
   filebeatdata01:
     driver: local
+  logstashdata01:
+    driver: local

elasticsearch/logstash01/Dockerfile

@@ -0,0 +1,9 @@
+ARG ELASTIC_STACK_VERSION
+
+FROM docker.elastic.co/logstash/logstash:${ELASTIC_STACK_VERSION}
+
+USER root
+
+COPY ./elasticsearch/logstash01/logstash.conf pipeline/logstash.conf
+
+RUN chmod go-w pipeline/logstash.conf

elasticsearch/logstash01/logstash.conf

@@ -0,0 +1,22 @@
+input {
+ file {
+   #https://www.elastic.co/guide/en/logstash/current/plugins-inputs-file.html
+   #default is TAIL which assumes more data will come into the file.
+   #change to mode => "read" if the file is a compelte file.  by default, the file will be removed once reading is complete -- backup your files if you need them.
+   mode => "tail"
+   path => "/usr/share/logstash/ingest_data/*"
+ }
+}
+
+filter {
+}
+
+output {
+ elasticsearch {
+   index => "logstash-%{+YYYY.MM.dd}"
+   hosts=> "${ELASTIC_HOSTS}"
+   user=> "${ELASTIC_USER}"
+   password=> "${ELASTIC_PASSWORD}"
+   cacert=> "certs/ca/ca.crt"
+ }
+}

elasticsearch/reset.sh

@@ -5,4 +5,4 @@ docker compose down
 
 docker volume ls -q | grep pems_ | xargs docker volume rm
 
-docker compose build metricbeat01 filebeat01
+docker compose build metricbeat01 filebeat01 logstash01