Pre-build logstash image with plugin and configuration (#1687)

Polish some things in the logstash scenario: * Pre-build logstash image: * It includes the plugin now so it doesn't need to be installed on every run, speeding up startup after first run. * It includes the configuration, so it can be modified with auto-reload, as introduced in #1668. * Use private key without converting it to PKCS8. * Use actual CA instead of elasticsearch certificate as CA in logstash output. * Startup script removed as operations moved to other places or not needed. * Use the same configuration template for serverless and compose providers.
elastic · Feb 19, 2024 · 26e81fe · 26e81fe
1 parent a9b043b
commit 26e81fe
Show file tree

Hide file tree

Showing 8 changed files with 53 additions and 113 deletions.
diff --git a/internal/stack/_static/Dockerfile.logstash b/internal/stack/_static/Dockerfile.logstash
@@ -0,0 +1,13 @@
+ARG IMAGE
+FROM ${IMAGE}
+
+RUN if [ ! "$(bin/logstash-plugin list)" = *logstash-filter-elastic_integration* ]; then \
+        echo "Missing plugin logstash-filter-elastic_integration, installing now."; \
+        bin/logstash-plugin install logstash-filter-elastic_integration; \
+    fi
+
+# Copying the file here instead of using a volume so it can be modified and reload
+# automatically.
+COPY --chown=logstash:root ./logstash.conf /usr/share/logstash/pipeline/logstash.conf
+
+CMD bin/logstash -f /usr/share/logstash/pipeline/logstash.conf --config.reload.automatic
diff --git a/internal/stack/_static/docker-compose-stack.yml.tmpl b/internal/stack/_static/docker-compose-stack.yml.tmpl
@@ -159,23 +159,21 @@ services:
 {{ $logstash_enabled := fact "logstash_enabled" }}
 {{ if eq $logstash_enabled "true" }}
   logstash:
+    build:
+      dockerfile: "./Dockerfile.logstash"
+      args:
+        IMAGE: "${LOGSTASH_IMAGE_REF}"
     depends_on:
       elasticsearch:
         condition: service_healthy
-      kibana:
-        condition: service_healthy
-    image: ${LOGSTASH_IMAGE_REF}
     healthcheck:
       test: bin/logstash -t
+      start_period: 120s
       interval: 60s
-      timeout: 50s
+      timeout: 60s
       retries: 5
-    command: bash /usr/share/logstash/startup.sh
     volumes:
       - "../certs/logstash:/usr/share/logstash/config/certs"
-      - "../certs/elasticsearch/cert.pem:/usr/share/logstash/config/certs/elasticsearch.pem:ro"
-      - "./logstash.conf:/usr/share/logstash/pipeline/generated_logstash.conf:ro"
-      - "./logstash_startup.sh:/usr/share/logstash/startup.sh"
     ports:
        - "127.0.0.1:5044:5044"
        - "127.0.0.1:9600:9600"

diff --git a/internal/stack/_static/logstash.conf.tmpl b/internal/stack/_static/logstash.conf.tmpl
@@ -4,30 +4,31 @@ input {
     ssl_enabled => true
     ssl_certificate_authorities => ["/usr/share/logstash/config/certs/ca-cert.pem"]
     ssl_certificate => "/usr/share/logstash/config/certs/cert.pem"
-    ssl_key => "/tmp/logstash.pkcs8.key"
+    ssl_key => "/usr/share/logstash/config/certs/key.pem"
   }
 }
 
-
+{{ $elasticsearch_host := fact "elasticsearch_host" -}}
 filter {
   elastic_integration {
     remove_field => ['@version']
-    hosts => ["https://elasticsearch:9200"]
-    username => {{ fact "username" }}
-    password => {{ fact "password" }}
+    hosts => ["{{ $elasticsearch_host }}"]
+    username => '{{ fact "username" }}'
+    password => '{{ fact "password" }}'
     ssl_enabled => true
     ssl_verification_mode => "none"
   }
 }
 
-
 output {
   elasticsearch {
-    hosts => ["https://elasticsearch:9200"]
-    user => {{ fact "username" }}
-    password => {{ fact "password" }}
+    hosts => ["{{ $elasticsearch_host }}"]
+    user => '{{ fact "username" }}'
+    password => '{{ fact "password" }}'
     ssl_enabled => true
-    ssl_certificate_authorities => "/usr/share/logstash/config/certs/elasticsearch.pem"
+{{- if eq $elasticsearch_host "https://elasticsearch:9200" }}
+    ssl_certificate_authorities => "/usr/share/logstash/config/certs/ca-cert.pem"
     document_id => "%{[@metadata][_ingest_document][id]}"
+{{- end }}
   }
 }
diff --git a/internal/stack/_static/logstash_startup.sh b/internal/stack/_static/logstash_startup.sh
diff --git a/internal/stack/_static/serverless-docker-compose.yml.tmpl b/internal/stack/_static/serverless-docker-compose.yml.tmpl
@@ -29,17 +29,18 @@ services:
 {{ $logstash_enabled := fact "logstash_enabled" }}
 {{ if eq $logstash_enabled "true" }}
   logstash:
-    image: "{{ fact "logstash_image" }}"
+    build:
+      dockerfile: "./Dockerfile.logstash"
+      args:
+        IMAGE: "{{ fact "logstash_image" }}"
     healthcheck:
       test: bin/logstash -t
+      start_period: 120s
       interval: 60s
-      timeout: 50s
+      timeout: 60s
       retries: 5
-    command: bash /usr/share/logstash/startup.sh
     volumes:
       - "../certs/logstash:/usr/share/logstash/config/certs"
-      - "./logstash.conf:/usr/share/logstash/pipeline/generated_logstash.conf:ro"
-      - "./logstash_startup.sh:/usr/share/logstash/startup.sh"
     ports:
        - "127.0.0.1:5044:5044"
        - "127.0.0.1:9600:9600"

diff --git a/internal/stack/_static/serverless-logstash.conf.tmpl b/internal/stack/_static/serverless-logstash.conf.tmpl
diff --git a/internal/stack/resources.go b/internal/stack/resources.go
@@ -105,10 +105,6 @@ var (
 			Path:    KibanaConfigFile,
 			Content: staticSource.Template("_static/kibana.yml.tmpl"),
 		},
-		&resource.File{
-			Path:    LogstashConfigFile,
-			Content: staticSource.Template("_static/logstash.conf.tmpl"),
-		},
 		&resource.File{
 			Path:    KibanaHealthcheckFile,
 			Content: staticSource.Template("_static/kibana_healthcheck.sh.tmpl"),
@@ -121,11 +117,16 @@ var (
 			Path:    ElasticAgentEnvFile,
 			Content: staticSource.Template("_static/elastic-agent.env.tmpl"),
 		},
+	}
+
+	logstashResources = []resource.Resource{
 		&resource.File{
-			Path:         "logstash_startup.sh",
-			CreateParent: true,
-			Content:      staticSource.Template("_static/logstash_startup.sh"),
-			Mode:         resource.FileMode(0755),
+			Path:    LogstashConfigFile,
+			Content: staticSource.Template("_static/logstash.conf.tmpl"),
+		},
+		&resource.File{
+			Path:    "Dockerfile.logstash",
+			Content: staticSource.File("_static/Dockerfile.logstash"),
 		},
 	}
 )
@@ -167,15 +168,16 @@ func applyResources(profile *profile.Profile, stackVersion string) error {
 	if err != nil {
 		return fmt.Errorf("failed to create TLS files: %w", err)
 	}
+	resources = append(resources, certResources...)
 
-	// Add client certificates if logstash is enabled
+	// Add related resources and client certificates if logstash is enabled.
 	if profile.Config("stack.logstash_enabled", "false") == "true" {
+		resources = append(resources, logstashResources...)
 		if err := addClientCertsToResources(resourceManager, certResources); err != nil {
 			return fmt.Errorf("error adding client certificates: %w", err)
 		}
 	}
 
-	resources = append(resources, certResources...)
 	results, err := resourceManager.Apply(resources)
 	if err != nil {
 		var errors []string

diff --git a/internal/stack/serverlessresources.go b/internal/stack/serverlessresources.go
@@ -28,16 +28,6 @@ var (
 			Path:    ElasticAgentEnvFile,
 			Content: staticSource.Template("_static/elastic-agent.env.tmpl"),
 		},
-		&resource.File{
-			Path:    LogstashConfigFile,
-			Content: staticSource.Template("_static/serverless-logstash.conf.tmpl"),
-		},
-		&resource.File{
-			Path:         "logstash_startup.sh",
-			CreateParent: true,
-			Content:      staticSource.Template("_static/logstash_startup.sh"),
-			Mode:         resource.FileMode(0755),
-		},
 	}
 )
 
@@ -79,6 +69,11 @@ func applyServerlessResources(profile *profile.Profile, stackVersion string, con
 	}
 	resources = append(resources, certResources...)
 
+	// Add related resources and client certificates if logstash is enabled.
+	if profile.Config("stack.logstash_enabled", "false") == "true" {
+		resources = append(resources, logstashResources...)
+	}
+
 	results, err := resourceManager.Apply(resources)
 	if err != nil {
 		var errors []string