Feat: local Elastic Stack setup (#23)

.devcontainer/devcontainer.json

@@ -2,8 +2,16 @@
   "name": "caltrans/pems",
   "dockerComposeFile": ["../compose.yml"],
   "service": "dev",
-  "runServices": ["dev", "docs"],
-  "forwardPorts": ["docs:8000"],
+  "runServices": [
+    "dev",
+    "docs",
+    "es01",
+    "kibana",
+    "metricbeat01",
+    "filebeat01",
+    "logstash01"
+  ],
+  "forwardPorts": ["docs:8000", "kibana:5601"],
   "workspaceFolder": "/home/caltrans/src",
   "postAttachCommand": ["/bin/bash", ".devcontainer/postAttach.sh"],
   "customizations": {

.env.sample

@@ -0,0 +1,23 @@
+# Version of Elastic products
+ELASTIC_STACK_VERSION=8.16.1
+
+# Set to 'basic' or 'trial' to automatically start the 30-day trial
+ELASTIC_LICENSE=basic
+
+# SAMPLE Predefined Key only to be used in POC environments
+ELASTIC_ENCRYPTION_KEY=c34d38b3a14956121ff2170e5030b471551370178f43e5626eec58b04a30fae2
+
+# Set the cluster name
+ELASTIC_CLUSTER=eslocal
+
+# Increase or decrease based on the available host memory (in bytes)
+ELASTIC_MEM_LIMIT=2147483648
+
+# Password for the 'elastic' user (at least 6 characters)
+ELASTIC_PASSWORD=elastic
+
+# Password for the 'kibana_system' user (at least 6 characters)
+KIBANA_PASSWORD=kibana
+
+# Increase or decrease based on the available host memory (in bytes)
+KIBANA_MEM_LIMIT=1073741824

compose.yml

@@ -17,3 +17,185 @@ services:
       - "8000"
     volumes:
       - ./:/home/caltrans/src
+
+  esconfig:
+    image: docker.elastic.co/elasticsearch/elasticsearch:${ELASTIC_STACK_VERSION}
+    user: "0"
+    command: ["bash", "/.local/config/setup.sh"]
+    env_file:
+      - .env
+    volumes:
+      - certs:/usr/share/elasticsearch/config/certs
+      - ./elasticstack/esconfig:/.local/config
+    healthcheck:
+      test: ["CMD-SHELL", "[ -f config/certs/es01/es01.crt ]"]
+      interval: 1s
+      timeout: 5s
+      retries: 120
+
+  es01:
+    depends_on:
+      esconfig:
+        condition: service_healthy
+    image: docker.elastic.co/elasticsearch/elasticsearch:${ELASTIC_STACK_VERSION}
+    labels:
+      co.elastic.logs/module: elasticsearch
+    volumes:
+      - certs:/usr/share/elasticsearch/config/certs
+      - esdata01:/usr/share/elasticsearch/data
+    ports:
+      - "9200"
+    environment:
+      - node.name=es01
+      - cluster.name=${ELASTIC_CLUSTER}
+      - discovery.type=single-node
+      - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
+      - bootstrap.memory_lock=true
+      - xpack.security.enabled=true
+      - xpack.security.http.ssl.enabled=true
+      - xpack.security.http.ssl.key=certs/es01/es01.key
+      - xpack.security.http.ssl.certificate=certs/es01/es01.crt
+      - xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt
+      - xpack.security.transport.ssl.enabled=true
+      - xpack.security.transport.ssl.key=certs/es01/es01.key
+      - xpack.security.transport.ssl.certificate=certs/es01/es01.crt
+      - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt
+      - xpack.security.transport.ssl.verification_mode=certificate
+      - xpack.license.self_generated.type=${ELASTIC_LICENSE}
+    mem_limit: ${ELASTIC_MEM_LIMIT}
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+    healthcheck:
+      test:
+        [
+          "CMD-SHELL",
+          "curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication credentials'",
+        ]
+      interval: 10s
+      timeout: 10s
+      retries: 120
+
+  kibana:
+    depends_on:
+      es01:
+        condition: service_healthy
+    image: docker.elastic.co/kibana/kibana:${ELASTIC_STACK_VERSION}
+    labels:
+      co.elastic.logs/module: kibana
+    volumes:
+      - certs:/usr/share/kibana/config/certs
+      - kibanadata:/usr/share/kibana/data
+      - ./elasticstack/kibana:/usr/share/kibana/config
+    ports:
+      - "5601"
+    environment:
+      - SERVERNAME=kibana
+      - ELASTICSEARCH_HOSTS=https://es01:9200
+      - ELASTICSEARCH_USERNAME=kibana_system
+      - ELASTICSEARCH_PASSWORD=${KIBANA_PASSWORD}
+      - ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES=config/certs/ca/ca.crt
+      - XPACK_SECURITY_ENCRYPTIONKEY=${ELASTIC_ENCRYPTION_KEY}
+      - XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY=${ELASTIC_ENCRYPTION_KEY}
+      - XPACK_REPORTING_ENCRYPTIONKEY=${ELASTIC_ENCRYPTION_KEY}
+    mem_limit: ${KIBANA_MEM_LIMIT}
+    healthcheck:
+      test:
+        [
+          "CMD-SHELL",
+          "curl -s -I http://localhost:5601 | grep -q 'HTTP/1.1 302 Found'",
+        ]
+      interval: 10s
+      timeout: 10s
+      retries: 120
+
+  metricbeat01:
+    depends_on:
+      es01:
+        condition: service_healthy
+      kibana:
+        condition: service_healthy
+    image: caltrans/pems:metricbeat01
+    build:
+      context: .
+      dockerfile: ./elasticstack/metricbeat01/Dockerfile
+      args:
+        - ELASTIC_STACK_VERSION=${ELASTIC_STACK_VERSION}
+    volumes:
+      - certs:/usr/share/metricbeat/certs
+      - metricbeatdata01:/usr/share/metricbeat/data
+      - "/var/run/docker.sock:/var/run/docker.sock:ro"
+      - "/sys/fs/cgroup:/hostfs/sys/fs/cgroup:ro"
+      - "/proc:/hostfs/proc:ro"
+      - "/:/hostfs:ro"
+    environment:
+      - ELASTIC_USER=elastic
+      - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
+      - ELASTIC_HOSTS=https://es01:9200
+      - KIBANA_HOSTS=http://kibana:5601
+      - LOGSTASH_HOSTS=http://logstash01:9600
+    command:
+      - --strict.perms=false
+
+  filebeat01:
+    depends_on:
+      es01:
+        condition: service_healthy
+    image: caltrans/pems:filebeat01
+    build:
+      context: .
+      dockerfile: ./elasticstack/filebeat01/Dockerfile
+      args:
+        - ELASTIC_STACK_VERSION=${ELASTIC_STACK_VERSION}
+    volumes:
+      - certs:/usr/share/filebeat/certs
+      - filebeatdata01:/usr/share/filebeat/data
+      - "/var/lib/docker/containers:/var/lib/docker/containers:ro"
+      - "/var/run/docker.sock:/var/run/docker.sock:ro"
+    environment:
+      - ELASTIC_USER=elastic
+      - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
+      - ELASTIC_HOSTS=https://es01:9200
+      - KIBANA_HOSTS=http://kibana:5601
+      - LOGSTASH_HOSTS=http://logstash01:9600
+    command:
+      - --strict.perms=false
+
+  logstash01:
+    depends_on:
+      es01:
+        condition: service_healthy
+      kibana:
+        condition: service_healthy
+    image: caltrans/pems:logstash01
+    build:
+      context: .
+      dockerfile: ./elasticstack/logstash01/Dockerfile
+      args:
+        - ELASTIC_STACK_VERSION=${ELASTIC_STACK_VERSION}
+    labels:
+      co.elastic.logs/module: logstash
+    volumes:
+      - certs:/usr/share/logstash/certs
+      - logstashdata01:/usr/share/logstash/data
+      - ./elasticstack/logstash01:/tmp/logstash01
+    environment:
+      - xpack.monitoring.enabled=false
+      - ELASTIC_USER=elastic
+      - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
+      - ELASTIC_HOSTS=https://es01:9200
+
+volumes:
+  certs:
+    driver: local
+  esdata01:
+    driver: local
+  kibanadata:
+    driver: local
+  metricbeatdata01:
+    driver: local
+  filebeatdata01:
+    driver: local
+  logstashdata01:
+    driver: local

elasticstack/esconfig/instances.yml

@@ -0,0 +1,13 @@
+instances:
+  - name: es01
+    dns:
+      - es01
+      - localhost
+    ip:
+      - 127.0.0.1
+  - name: kibana
+    dns:
+      - kibana
+      - localhost
+    ip:
+      - 127.0.0.1

elasticstack/esconfig/setup.sh

@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+set -eu
+
+if [ x${ELASTIC_PASSWORD} == x ]; then
+    echo "Set the ELASTIC_PASSWORD environment variable in the .env file";
+    exit 1;
+elif [ x${KIBANA_PASSWORD} == x ]; then
+    echo "Set the KIBANA_PASSWORD environment variable in the .env file";
+    exit 1;
+fi;
+
+if [ ! -f config/certs/ca.zip ]; then
+    echo "Creating CA";
+    bin/elasticsearch-certutil ca --silent --pem -out config/certs/ca.zip;
+    unzip config/certs/ca.zip -d config/certs;
+fi;
+
+if [ ! -f config/certs/certs.zip ]; then
+    echo "Creating certs";
+    cp /.local/config/instances.yml config/certs/instances.yml;
+    bin/elasticsearch-certutil cert --silent --pem \
+        -out config/certs/certs.zip \
+        --in config/certs/instances.yml \
+        --ca-cert config/certs/ca/ca.crt \
+        --ca-key config/certs/ca/ca.key;
+    unzip config/certs/certs.zip -d config/certs;
+fi;
+
+echo "Setting file permissions"
+chown -R root:root config/certs;
+
+find . -type d -exec chmod 755 \{\} \;;
+find . -type f -exec chmod 644 \{\} \;;
+
+TIMEOUT=10
+
+until
+    echo "Waiting for Elasticsearch availability (sleeping for ${TIMEOUT}s)";
+    curl -s --cacert config/certs/ca/ca.crt https://es01:9200 | grep -q "missing authentication credentials";
+do sleep $TIMEOUT; done;
+
+until
+    echo "Setting kibana_system password (sleeping for ${TIMEOUT}s)";
+    curl -s -X POST \
+        --cacert config/certs/ca/ca.crt \
+        -u "elastic:${ELASTIC_PASSWORD}" \
+        -H "Content-Type: application/json" https://es01:9200/_security/user/kibana_system/_password \
+        -d "{\"password\":\"${KIBANA_PASSWORD}\"}" | grep -q "^{}";
+do sleep $TIMEOUT; done;
+
+echo "All done!";

elasticstack/filebeat01/Dockerfile

@@ -0,0 +1,9 @@
+ARG ELASTIC_STACK_VERSION=8.16.1
+
+FROM docker.elastic.co/beats/filebeat:${ELASTIC_STACK_VERSION}
+
+USER root
+
+COPY ./elasticstack/filebeat01/filebeat.yml filebeat.yml
+
+RUN chmod go-w filebeat.yml

elasticstack/filebeat01/filebeat.yml

@@ -0,0 +1,24 @@
+filebeat.autodiscover:
+  providers:
+    - type: docker
+      hints.enabled: true
+      hints.default_config:
+        type: container
+        paths:
+          - /var/lib/docker/containers/${data.container.id}/*.log
+
+processors:
+  - add_docker_metadata: ~
+
+setup.kibana:
+  host: ${KIBANA_HOSTS}
+  username: ${ELASTIC_USER}
+  password: ${ELASTIC_PASSWORD}
+
+output.elasticsearch:
+  hosts: ${ELASTIC_HOSTS}
+  username: ${ELASTIC_USER}
+  password: ${ELASTIC_PASSWORD}
+  ssl:
+    enabled: true
+    certificate_authorities: certs/ca/ca.crt

elasticstack/kibana/kibana.yml

@@ -0,0 +1,2 @@
+server.host: "0.0.0.0"
+telemetry.optIn: "false"

elasticstack/logstash01/Dockerfile

@@ -0,0 +1,9 @@
+ARG ELASTIC_STACK_VERSION=8.16.1
+
+FROM docker.elastic.co/logstash/logstash:${ELASTIC_STACK_VERSION}
+
+USER root
+
+COPY ./elasticstack/logstash01/logstash.conf pipeline/logstash.conf
+
+RUN chmod go-w pipeline/logstash.conf

elasticstack/logstash01/logstash.conf

@@ -0,0 +1,20 @@
+input {
+ file {
+    # https://www.elastic.co/guide/en/logstash/current/plugins-inputs-file.html
+    mode => "read"
+    path => "/tmp/logstash01/*.log"
+ }
+}
+
+filter {
+}
+
+output {
+ elasticsearch {
+   index => "logstash-%{+YYYY.MM.dd}"
+   hosts=> "${ELASTIC_HOSTS}"
+   user=> "${ELASTIC_USER}"
+   password=> "${ELASTIC_PASSWORD}"
+   ssl_certificate_authorities=> "certs/ca/ca.crt"
+ }
+}

elasticstack/metricbeat01/Dockerfile

@@ -0,0 +1,9 @@
+ARG ELASTIC_STACK_VERSION=8.16.1
+
+FROM docker.elastic.co/beats/metricbeat:${ELASTIC_STACK_VERSION}
+
+USER root
+
+COPY ./elasticstack/metricbeat01/metricbeat.yml metricbeat.yml
+
+RUN chmod go-w metricbeat.yml