Improve Kibana docker image build (#5966)

* Add `NODE_OPTIONS="--max-old-space-size=4096"` to circumvent elastic/kibana#15683 * Make it more obvious in the Docker file which parts are copied over and which are from the official repo * Remove xpack.monitoring config option as this seems to cause a rebuild. * Increase LS timeout for health check * Remove duplicated kafka health check * Increase health check timeouts to 10min * Set defaults for `ARGS` in kibana container. This makes it easy to also build the container directly. * Adjusts docker image paths to all use `ES_BEATS` variable. * Remove `detector_rules` for ML as not supported / required anymore in master. * Remove unused `SHIELD=false` variable in compose file * Fix geoip looksup that changed and improve error logging It would be great if we could disable most of the Kibana x-pack plugins as they are not needed for our testing. But the problem is 1) we cannot install only 1 plugin AFAIK 2) disabling one of the plugins causes a rebuild which makes the build even longer. Closes #5953
elastic · Jan 4, 2018 · e7f651e · e7f651e
1 parent ad812ec
commit e7f651e
Show file tree

Hide file tree

Showing 24 changed files with 70 additions and 59 deletions.
diff --git a/filebeat/docker-compose.yml b/filebeat/docker-compose.yml
@@ -28,12 +28,12 @@ services:
 
   elasticsearch:
     extends:
-      file: ../testing/environments/${TESTING_ENVIRONMENT}.yml
+      file: ${ES_BEATS}/testing/environments/${TESTING_ENVIRONMENT}.yml
       service: elasticsearch
 
   kibana:
     extends:
-      file: ../testing/environments/${TESTING_ENVIRONMENT}.yml
+      file: ${ES_BEATS}/testing/environments/${TESTING_ENVIRONMENT}.yml
       service: kibana
 
   redis:

diff --git a/filebeat/module/nginx/access/machine_learning/low_request_rate.json b/filebeat/module/nginx/access/machine_learning/low_request_rate.json
@@ -6,8 +6,7 @@
     "detectors": [
       {
         "detector_description": "nginx_access_low_request_rate",
-        "function": "low_count",
-        "detector_rules": []
+        "function": "low_count"
       }
     ],
     "influencers": []

diff --git a/filebeat/module/nginx/access/machine_learning/remote_ip_request_rate.json b/filebeat/module/nginx/access/machine_learning/remote_ip_request_rate.json
@@ -6,8 +6,7 @@
         {
         "detector_description": "nginx_access_remote_ip_high_count",
         "function": "high_count",
-        "over_field_name": "nginx.access.remote_ip",
-        "detector_rules": []
+        "over_field_name": "nginx.access.remote_ip"
       }
     ],
     "influencers": [

diff --git a/filebeat/module/nginx/access/machine_learning/remote_ip_url_count.json b/filebeat/module/nginx/access/machine_learning/remote_ip_url_count.json
@@ -7,8 +7,7 @@
         "detector_description": "nginx_access_remote_ip_high_dc_url",
         "function": "high_distinct_count",
         "field_name": "nginx.access.url",
-        "over_field_name": "nginx.access.remote_ip",
-        "detector_rules": []
+        "over_field_name": "nginx.access.remote_ip"
       }
     ],
     "influencers": [

diff --git a/filebeat/module/nginx/access/machine_learning/response_code.json b/filebeat/module/nginx/access/machine_learning/response_code.json
@@ -6,8 +6,7 @@
         {
         "detector_description": "nginx_access_response_code_rate",
         "function": "count",
-        "partition_field_name": "nginx.access.response_code",
-        "detector_rules": []
+        "partition_field_name": "nginx.access.response_code"
       }
     ],
     "influencers": [

diff --git a/filebeat/module/nginx/access/machine_learning/visitor_rate.json b/filebeat/module/nginx/access/machine_learning/visitor_rate.json
@@ -6,8 +6,7 @@
     "detectors": [
       {
         "detector_description": "nginx_access_visitor_rate",
-        "function": "non_zero_count",
-        "detector_rules": []
+        "function": "non_zero_count"
       }
     ],
     "influencers": []

diff --git a/filebeat/module/nginx/access/test/test.log-expected.json b/filebeat/module/nginx/access/test/test.log-expected.json
@@ -16,9 +16,11 @@
             "continent_name" : "Europe",
             "country_iso_code" : "DE",
             "location" : {
-              "lon" : 9.0,
-              "lat" : 51.0
-            }
+              "lon" : 13.4,
+              "lat" : 52.5167
+            },
+            "region_name": "Land Berlin",
+            "city_name": "Berlin"
           },
           "method" : "GET",
           "user_name" : "-",
@@ -127,9 +129,11 @@
             "continent_name" : "Europe",
             "country_iso_code" : "DE",
             "location" : {
-              "lon" : 9.0,
-              "lat" : 51.0
-            }
+              "lon" : 13.4,
+              "lat" : 52.5167
+            },
+            "region_name": "Land Berlin",
+            "city_name": "Berlin"
           },
           "method" : "GET",
           "user_name" : "-",
@@ -299,8 +303,8 @@
             "continent_name" : "Europe",
             "country_iso_code" : "PT",
             "location" : {
-              "lon" : -8.13057,
-              "lat" : 39.6945
+              "lon" : -8.0,
+              "lat" : 39.5
             }
           },
           "method" : "GET",

diff --git a/filebeat/module/system/auth/test/test.log-expected.json b/filebeat/module/system/auth/test/test.log-expected.json
@@ -299,12 +299,12 @@
                     "ssh": {
                         "geoip": {
                             "continent_name": "Asia",
-                            "city_name": "Guangzhou",
+                            "city_name": "Shenzhen",
                             "country_iso_code": "CN",
                             "region_name": "Guangdong",
                             "location": {
-                                "lon": 113.25,
-                                "lat": 23.1167
+                                "lon": 114.1333,
+                                "lat": 22.5333
                             }
                         },
                         "method": "password",

diff --git a/filebeat/module/traefik/access/machine_learning/low_request_rate.json b/filebeat/module/traefik/access/machine_learning/low_request_rate.json
@@ -6,8 +6,7 @@
     "detectors": [
       {
         "detector_description": "traefik_access_low_request_rate",
-        "function": "low_count",
-        "detector_rules": []
+        "function": "low_count"
       }
     ],
     "influencers": []

diff --git a/filebeat/module/traefik/access/machine_learning/remote_ip_request_rate.json b/filebeat/module/traefik/access/machine_learning/remote_ip_request_rate.json
@@ -6,8 +6,7 @@
         {
         "detector_description": "traefik_access_remote_ip_high_count",
         "function": "high_count",
-        "over_field_name": "traefik.access.remote_ip",
-        "detector_rules": []
+        "over_field_name": "traefik.access.remote_ip"
       }
     ],
     "influencers": [

diff --git a/filebeat/module/traefik/access/machine_learning/remote_ip_url_count.json b/filebeat/module/traefik/access/machine_learning/remote_ip_url_count.json
@@ -7,8 +7,7 @@
         "detector_description": "traefik_access_remote_ip_high_dc_url",
         "function": "high_distinct_count",
         "field_name": "traefik.access.url",
-        "over_field_name": "traefik.access.remote_ip",
-        "detector_rules": []
+        "over_field_name": "traefik.access.remote_ip"
       }
     ],
     "influencers": [

diff --git a/filebeat/module/traefik/access/machine_learning/response_code.json b/filebeat/module/traefik/access/machine_learning/response_code.json
@@ -6,8 +6,7 @@
         {
         "detector_description": "traefik_access_response_code_rate",
         "function": "count",
-        "partition_field_name": "traefik.access.response_code",
-        "detector_rules": []
+        "partition_field_name": "traefik.access.response_code"
       }
     ],
     "influencers": [

diff --git a/filebeat/module/traefik/access/machine_learning/visitor_rate.json b/filebeat/module/traefik/access/machine_learning/visitor_rate.json
@@ -6,8 +6,7 @@
     "detectors": [
       {
         "detector_description": "traefik_access_visitor_rate",
-        "function": "non_zero_count",
-        "detector_rules": []
+        "function": "non_zero_count"
       }
     ],
     "influencers": []

diff --git a/filebeat/tests/system/test_modules.py b/filebeat/tests/system/test_modules.py
@@ -89,7 +89,9 @@ def _test_expected_events(self, module, test_file, res, objects):
                 if ev["_source"][module] == obj[module]:
                     found = True
                     break
-            assert found, "The following expected object was not found: {}".format(obj)
+
+            assert found, "The following expected object was not found:\n {}\nSearched in: \n{}".format(
+                ev["_source"][module], objects)
 
     def run_on_file(self, module, fileset, test_file, cfgfile):
         print("Testing {}/{} on {}".format(module, fileset, test_file))

diff --git a/libbeat/docker-compose.yml b/libbeat/docker-compose.yml
@@ -14,8 +14,6 @@ services:
       - LS_TLS_PORT=5055
       # Setup work environment
       - LIBBEAT_PATH=/go/src/github.com/elastic/beats/libbeat
-      # TODO: Set to true as soon as stable release is out
-      - SHIELD=false
       - KAFKA_HOST=kafka
       - KAFKA_PORT=9092
       - KIBANA_HOST=kibana
@@ -39,18 +37,20 @@ services:
       kafka:         { condition: service_healthy }
       redis:         { condition: service_healthy }
       sredis:        { condition: service_healthy }
-      kafka:         { condition: service_healthy }
       kibana:        { condition: service_healthy }
+    healthcheck:
+      interval: 1s
+      retries: 600
 
   elasticsearch:
     extends:
-      file: ../testing/environments/${TESTING_ENVIRONMENT}.yml
+      file: ${ES_BEATS}/testing/environments/${TESTING_ENVIRONMENT}.yml
       service: elasticsearch
 
   # This host name is static because of the certificate.
   logstash:
     extends:
-      file: ../testing/environments/${TESTING_ENVIRONMENT}.yml
+      file: ${ES_BEATS}/testing/environments/${TESTING_ENVIRONMENT}.yml
       service: logstash
     env_file:
       - ${PWD}/build/test.env
@@ -59,11 +59,11 @@ services:
         condition: service_healthy
 
   redis:
-    build: ${PWD}/../testing/environments/docker/redis
+    build: ${ES_BEATS}/testing/environments/docker/redis
 
   # This host name is static because of the certificate.
   sredis: # stunnel proxy for redis
-    build: ${PWD}/../testing/environments/docker/sredis
+    build: ${ES_BEATS}/testing/environments/docker/sredis
     expose:
       - 6380
     depends_on:
@@ -76,7 +76,7 @@ services:
       - ${PWD}/build/test.env
 
   kafka:
-    build: ${PWD}/../testing/environments/docker/kafka
+    build: ${ES_BEATS}/testing/environments/docker/kafka
     expose:
       - 9092
       - 2181

diff --git a/libbeat/ml-importer/importer_integration_test.go b/libbeat/ml-importer/importer_integration_test.go
@@ -24,7 +24,6 @@ const sampleJob = `
       {
         "detector_description": "Event rate for nginx.access.response_code",
         "function": "count",
-        "detector_rules": [],
         "partition_field_name": "nginx.access.response_code"
       }
     ],

diff --git a/testing/environments/args.yml b/testing/environments/args.yml
@@ -7,4 +7,4 @@ services:
       args:
         DOWNLOAD_URL: https://snapshots.elastic.co/downloads
         ELASTIC_VERSION: 7.0.0-alpha1-SNAPSHOT
-        CACHE_BUST: 20171201
+        CACHE_BUST: 20180103
diff --git a/testing/environments/docker/elasticsearch/Dockerfile-snapshot b/testing/environments/docker/elasticsearch/Dockerfile-snapshot
@@ -55,6 +55,6 @@ RUN chown elasticsearch:elasticsearch config/elasticsearch.yml config/log4j2.pro
 USER elasticsearch
 CMD ["/bin/bash", "bin/es-docker"]
 
-HEALTHCHECK CMD curl -f http://localhost:9200
+HEALTHCHECK --interval=1s --retries=600 CMD curl -f http://localhost:9200
 
 EXPOSE 9200 9300
diff --git a/testing/environments/docker/kafka/Dockerfile b/testing/environments/docker/kafka/Dockerfile
@@ -21,6 +21,6 @@ EXPOSE 9092
 EXPOSE 2181
 
 # Healtcheck creates an empty topic foo. As soon as a topic is created, it assumes broke is available
-HEALTHCHECK --interval=1s --retries=90 CMD /healthcheck.sh
+HEALTHCHECK --interval=1s --retries=600 CMD /healthcheck.sh
 
 ENTRYPOINT ["/run.sh"]
diff --git a/testing/environments/docker/kibana/Dockerfile-snapshot b/testing/environments/docker/kibana/Dockerfile-snapshot
@@ -1,19 +1,22 @@
 # Copy of https://github.com/elastic/kibana-docker/blob/master/build/kibana/Dockerfile
 FROM centos:7
 LABEL maintainer "Elastic Docker Team <docker@elastic.co>"
-
 EXPOSE 5601
 
-ARG DOWNLOAD_URL
-ARG ELASTIC_VERSION
+
+### Beats specific args ####
+ARG DOWNLOAD_URL=https://snapshots.elastic.co/downloads
+ARG ELASTIC_VERSION=7.0.0-alpha1-SNAPSHOT
 ARG CACHE_BUST=1
-ARG XPACK
+ARG XPACK=1
 
-# Add Reporting dependencies + healthcheck tool
-RUN yum update -y && yum install -y epel-release
-RUN yum install -y fontconfig freetype jq && yum clean all
+# Healthcheck create by beats team
+RUN yum install update -y epel-release && yum install -y jq
+HEALTHCHECK --interval=1s --retries=600 CMD curl -f http://localhost:5601/api/status | jq '. | .status.overall.state' | grep -q green
+### Beats ###
 
-HEALTHCHECK --interval=1s --retries=300 CMD curl -f http://localhost:5601/api/status | jq '. | .status.overall.state' | grep -q green
+# Add Reporting dependencies + healthcheck tool
+RUN yum update -y && yum install -y fontconfig freetype && yum clean all
 
 WORKDIR /usr/share/kibana
 RUN curl -Ls ${DOWNLOAD_URL}/kibana/kibana-${ELASTIC_VERSION}-linux-x86_64.tar.gz?c=${CACHE_BUST} | tar --strip-components=1 -zxf - && \
@@ -39,6 +42,8 @@ RUN groupadd --gid 1000 kibana && \
       kibana
 USER kibana
 
-RUN if [ ${XPACK} = "1" ]; then bin/kibana-plugin install ${DOWNLOAD_URL}/kibana-plugins/x-pack/x-pack-${ELASTIC_VERSION}.zip?c=${CACHE_BUST}; fi
+# Beats specific check for XPACK to have both variables in one
+RUN if [ ${XPACK} = "1" ]; then NODE_OPTIONS="--max-old-space-size=4096" bin/kibana-plugin install ${DOWNLOAD_URL}/kibana-plugins/x-pack/x-pack-${ELASTIC_VERSION}.zip?c=${CACHE_BUST}; fi
 
 CMD ["/bin/bash", "/usr/local/bin/kibana-docker"]
+
diff --git a/testing/environments/docker/kibana/config/kibana-x-pack.yml b/testing/environments/docker/kibana/config/kibana-x-pack.yml
@@ -6,4 +6,15 @@ server.host: "0"
 elasticsearch.url: http://elasticsearch:9200
 elasticsearch.username: elastic
 elasticsearch.password: changeme
-xpack.monitoring.ui.container.elasticsearch.enabled: true
+
+# The following config options are currently commented out as they would cause a rebuild of kibana which takes a long time.
+
+#xpack.monitoring.ui.container.elasticsearch.enabled: true
+
+# Potentially disabled for beats testing when rebuilding is not needed
+#xpack.apm.enabled: false
+#xpack.graph.enabled: false
+#xpack.ml.enabled: false
+#xpack.monitoring.enabled: false
+#xpack.reporting.enabled: false
+#xpack.watcher.enabled: false
diff --git a/testing/environments/docker/logstash/Dockerfile b/testing/environments/docker/logstash/Dockerfile
@@ -22,7 +22,7 @@ COPY docker-entrypoint.sh /entrypoint.sh
 
 COPY pki /etc/pki
 
-HEALTHCHECK CMD nc -z localhost 5044
+HEALTHCHECK --interval=1s --retries=600 CMD nc -z localhost 5044
 
 ENTRYPOINT ["/entrypoint.sh"]
 

diff --git a/testing/environments/docker/redis/Dockerfile b/testing/environments/docker/redis/Dockerfile
@@ -1,2 +1,2 @@
 FROM redis:3.2.4-alpine
-HEALTHCHECK CMD nc -z localhost 6379
+HEALTHCHECK --interval=1s --retries=600 CMD nc -z localhost 6379
diff --git a/testing/environments/docker/sredis/Dockerfile b/testing/environments/docker/sredis/Dockerfile
@@ -9,7 +9,7 @@ RUN chmod 600 /etc/stunnel/stunnel.conf; \
 	chmod 600 /etc/pki/tls/certs/*; \
 	chmod 600 /etc/pki/tls/private/*;
 
-HEALTHCHECK CMD nc -z localhost 6380
+HEALTHCHECK --interval=1s --retries=600 CMD nc -z localhost 6380
 EXPOSE 6380
 
 CMD ["stunnel"]