Filebeat upgrade to v7.8.1

CHANGELOG-0.7.md

@@ -1,11 +1,13 @@
 # Changelog 0.7
 
-## [0.7.1] 2020-07-xx
+## [0.7.1] 2020-08-12
 
 ### Added
 
 - Minor logging improvements added while fixing issue [#1424](https://github.com/epiphany-platform/epiphany/issues/1424)
 - [#1438](https://github.com/epiphany-platform/epiphany/pull/1438) - Rename Terraform plugin vendor in VSCode recommendations
+- [#1413](https://github.com/epiphany-platform/epiphany/issues/1413) - Set protocol for Vault only in one place in configuration
+- [#1423](https://github.com/epiphany-platform/epiphany/issues/1423) - Error reading generated service principal
 
 ### Updated
 
@@ -28,11 +30,6 @@
 - [#1336](https://github.com/epiphany-platform/epiphany/issues/1336) - Deployment of version 0.7.0 failed on-prem (spec.hostname)
 - [#1394](https://github.com/epiphany-platform/epiphany/issues/1394) - Cannot access Kubernetes dashboard after upgrading
 
-### Added
-
-- [#1413](https://github.com/epiphany-platform/epiphany/issues/1413) - Set protocol for Vault only in one place in configuration
-- [#1423](https://github.com/epiphany-platform/epiphany/issues/1423) - Error reading generated service principal
-
 ## [0.7.0] 2020-06-30
 
 ### Added

CHANGELOG-0.8.md

@@ -0,0 +1,11 @@
+# Changelog 0.8
+
+## [0.8.0] 2020-09-xx
+
+### Added
+
+### Updated
+
+- [#846](https://github.com/epiphany-platform/epiphany/issues/846) - Update Filebeat to v7.8.1
+
+### Fixed

README.md

@@ -50,10 +50,10 @@ This minimum file definition is fine to start with, if you need more control ove
 epicli init -p aws -n demo --full
 ```
 
-You will need to modify a few values (like your AWS secrets, directory path for SSH keys). Once you are done with `demo.yaml` you can start cluster deployment by executing:
+You will need to modify a few values (like your AWS secrets, directory path for SSH keys). Once you are done with `demo.yml` you can start cluster deployment by executing:
 
 ```shell
-epicli apply -f demo.yaml
+epicli apply -f demo.yml
 ```
 You will be asked for a password that will be used for encryption of some of build artifacts. More information [here](docs/home/howto/SECURITY.md#how-to-run-epicli-with-password)
 
@@ -63,6 +63,11 @@ epicli backup -f <file.yml> -b <build_folder>
 epicli recovery -f <file.yml> -b <build_folder>
 ```
 
+To delete all deployed components following command should be used
+
+```shell
+epicli delete -b <build_folder>
+```
 
 Find more information using table of contents below - especially the [How-to guides](docs/home/HOWTO.md).
 

core/src/epicli/cli/version.txt.py

@@ -1 +1 @@
-0.7.0
+0.7.1

core/src/epicli/data/common/ansible/playbooks/roles/filebeat/defaults/main.yml

@@ -1,3 +1,3 @@
 ---
 specification:
-  filebeat_version: "6.8.5"
+  filebeat_version: "7.8.1"

core/src/epicli/data/common/ansible/playbooks/roles/filebeat/templates/filebeat.yml.j2

@@ -19,7 +19,7 @@ filebeat.inputs:
 - type: log
   enabled: true
 
-  # Paths (in alphabetical order) that should be crawled and fetched. Glob based paths.
+  # Paths that should be crawled and fetched. Glob based paths.
   paths:
 #   - /var/log/audit/audit.log
     - /var/log/auth.log
@@ -34,7 +34,7 @@ filebeat.inputs:
     - /var/log/secure
     - /var/log/syslog
 
-  # Exclude lines. A list of regular expressions to match. It drops the lines that are
+ # Exclude lines. A list of regular expressions to match. It drops the lines that are
   # matching any regular expression from the list.
   #exclude_lines: ['^DBG']
 
@@ -67,9 +67,10 @@ filebeat.inputs:
   # that was (not) matched before or after or as long as a pattern is not matched based on negate.
   # Note: After is the equivalent to previous and before is the equivalent to to next in Logstash
   #multiline.match: after
+
 {% if 'postgresql' in group_names %}
 
-#--- PostgreSQL ---
+# ============================== PostgreSQL ==============================
 
 # Filebeat postgresql module doesn't support custom log_line_prefix (without patching), see https://discuss.elastic.co/t/filebeats-with-postgresql-module-custom-log-line-prefix/204457
 # Dedicated configuration to handle log messages spanning multiple lines.
@@ -85,9 +86,10 @@ filebeat.inputs:
     negate: true
     match: after
 {% endif %}
+
 {% if 'kubernetes_master' in group_names or 'kubernetes_node' in group_names %}
 
-#--- Kubernetes ---
+# ==============================  Kubernetes ==============================
 
 # K8s metadata are fetched from Docker labels to not make Filebeat on worker nodes dependent on K8s master
 # since Filebeat should start even if K8s master is not available.
@@ -112,7 +114,7 @@ filebeat.inputs:
           - docker # Drop all fields added by 'add_docker_metadata' that were not renamed
 {% endif %}
 
-#============================= Filebeat modules ===============================
+# ============================== Filebeat modules ==============================
 
 filebeat.config.modules:
   # Glob pattern for configuration loading
@@ -124,14 +126,14 @@ filebeat.config.modules:
   # Period on which files under path should be checked for changes
   #reload.period: 10s
 
-#==================== Elasticsearch template setting ==========================
+# ======================= Elasticsearch template setting =======================
 
 setup.template.settings:
   index.number_of_shards: 3
   #index.codec: best_compression
   #_source.enabled: false
 
-#================================ General =====================================
+# ================================== General ===================================
 
 # The name of the shipper that publishes the network data. It can be used to group
 # all the transactions sent by a single shipper in the web interface.
@@ -147,18 +149,54 @@ setup.template.settings:
 #  env: staging
 
 
-#============================== Dashboards =====================================
+# ================================= Dashboards =================================
 # These settings control loading the sample dashboards to the Kibana index. Loading
 # the dashboards is disabled by default and can be enabled either by setting the
-# options here, or by using the `-setup` CLI flag or the `setup` command.
-#setup.dashboards.enabled: true
+# options here or by using the `setup` command.
+#setup.dashboards.enabled: false
 
 # The URL from where to download the dashboards archive. By default this URL
 # has a value which is computed based on the Beat name and version. For released
 # versions, this URL points to the dashboard archive on the artifacts.elastic.co
 # website.
 #setup.dashboards.url:
 
+# ====================== Index Lifecycle Management (ILM) ======================
+
+# Configure index lifecycle management (ILM). These settings create a write
+# alias and add additional settings to the index template. When ILM is enabled,
+# output.elasticsearch.index is ignored, and the write alias is used to set the
+# index name.
+
+# Enable ILM support. Valid values are true, false, and auto. When set to auto
+# (the default), the Beat uses index lifecycle management when it connects to a
+# cluster that supports ILM; otherwise, it creates daily indices.
+# Disabled because ILM is not enabled by default in Epiphany
+setup.ilm.enabled: false
+
+# Set the prefix used in the index lifecycle write alias name. The default alias
+# name is 'filebeat-%{[agent.version]}'.
+#setup.ilm.rollover_alias: 'filebeat'
+
+# Set the rollover index pattern. The default is "%{now/d}-000001".
+#setup.ilm.pattern: "{now/d}-000001"
+
+# Set the lifecycle policy name. The default policy name is
+# 'beatname'.
+#setup.ilm.policy_name: "mypolicy"
+
+# The path to a JSON file that contains a lifecycle policy configuration. Used
+# to load your own lifecycle policy.
+#setup.ilm.policy_file:
+
+# Disable the check for an existing lifecycle policy. The default is true. If
+# you disable this check, set setup.ilm.overwrite: true so the lifecycle policy
+# can be installed.
+#setup.ilm.check_exists: true
+
+# Overwrite the lifecycle policy at startup. The default is false.
+#setup.ilm.overwrite: false
+
 #============================== Kibana =====================================
 
 # Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API.
@@ -182,9 +220,9 @@ setup.template.settings:
   # the Default Space will be used.
   #space.id:
 
-#============================= Elastic Cloud ==================================
+# =============================== Elastic Cloud ================================
 
-# These settings simplify using filebeat with the Elastic Cloud (https://cloud.elastic.co/).
+# These settings simplify using Filebeat with the Elastic Cloud (https://cloud.elastic.co/).
 
 # The cloud.id setting overwrites the `output.elasticsearch.hosts` and
 # `setup.kibana.host` options.
@@ -210,19 +248,24 @@ output.elasticsearch:
     - "https://{{hostvars[host]['ansible_hostname']}}:9200"
     {% endfor %}
 
+  # Protocol - either `http` (default) or `https`.
   protocol: "https"
   ssl.verification_mode: none
   username: logstash
   password: logstash
 {% else %}
   hosts: []
+  # Protocol - either `http` (default) or `https`.
   #protocol: "https"
+
   #ssl.verification_mode: none
+  # Authentication credentials - either API key or username/password.
+  #api_key: "id:api_key"
   #username: "elastic"
   #password: "changeme"
 {% endif %}
 
-#----------------------------- Logstash output --------------------------------
+# ------------------------------ Logstash Output -------------------------------
 #output.logstash:
   # The Logstash hosts
   #hosts: ["localhost:5044"]
@@ -237,15 +280,17 @@ output.elasticsearch:
   # Client Certificate Key
   #ssl.key: "/etc/pki/client/cert.key"
 
-#================================ Processors =====================================
+# ================================= Processors =================================
 
 # Configure processors to enhance or manipulate events generated by the beat.
 
 processors:
   #- add_host_metadata: ~
   - add_cloud_metadata: ~
+  #- add_docker_metadata: ~
+  #- add_kubernetes_metadata: ~
 
-#================================ Logging =====================================
+# ================================== Logging ===================================
 
 # Sets log level. The default log level is info.
 # Available log levels are: error, warning, info, debug
@@ -256,17 +301,30 @@ processors:
 # "publish", "service".
 #logging.selectors: ["*"]
 
-#============================== Xpack Monitoring ===============================
-# filebeat can export internal metrics to a central Elasticsearch monitoring
+# ============================= X-Pack Monitoring ==============================
+# Filebeat can export internal metrics to a central Elasticsearch monitoring
 # cluster.  This requires xpack monitoring to be enabled in Elasticsearch.  The
 # reporting is disabled by default.
 
 # Set to true to enable the monitoring reporter.
-#xpack.monitoring.enabled: false
+#monitoring.enabled: false
+
+# Sets the UUID of the Elasticsearch cluster under which monitoring data for this
+# Filebeat instance will appear in the Stack Monitoring UI. If output.elasticsearch
+# is enabled, the UUID is derived from the Elasticsearch cluster referenced by output.elasticsearch.
+#monitoring.cluster_uuid:
 
 # Uncomment to send the metrics to Elasticsearch. Most settings from the
-# Elasticsearch output are accepted here as well. Any setting that is not set is
-# automatically inherited from the Elasticsearch output configuration, so if you
-# have the Elasticsearch output configured, you can simply uncomment the
-# following line.
-#xpack.monitoring.elasticsearch:
+# Elasticsearch output are accepted here as well.
+# Note that the settings should point to your Elasticsearch *monitoring* cluster.
+# Any setting that is not set is automatically inherited from the Elasticsearch
+# output configuration, so if you have the Elasticsearch output configured such
+# that it is pointing to your Elasticsearch monitoring cluster, you can simply
+# uncomment the following line.
+#monitoring.elasticsearch:
+
+# ================================= Migration ==================================
+
+# Enable the compatibility layer for Elastic Common Schema (ECS) fields.
+# This allows to enable 6 > 7 migration aliases.
+#migration.6_to_7.enabled: true

core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_common/tasks/apiserver-certificates.yml

@@ -0,0 +1,34 @@
+---
+- name: Copy /etc/kubernetes/pki/apiserver.{crt,key}
+  copy:
+    dest: "{{ item }}.OLD"
+    src: "{{ item }}"
+    remote_src: true
+  loop:
+    - /etc/kubernetes/pki/apiserver.crt
+    - /etc/kubernetes/pki/apiserver.key
+
+- name: Delete /etc/kubernetes/pki/apiserver.{crt,key}
+  file:
+    path: "{{ item }}"
+    state: absent
+  loop:
+    - /etc/kubernetes/pki/apiserver.crt
+    - /etc/kubernetes/pki/apiserver.key
+
+- name: Render new certificates /etc/kubernetes/pki/apiserver.{crt,key}
+  shell: |
+    kubeadm init phase certs apiserver \
+      --config /etc/kubeadm/kubeadm-config.yml
+  args:
+    executable: /bin/bash
+    creates: /etc/kubernetes/pki/apiserver.key
+
+- name: Restart apiserver
+  shell: |
+    docker ps \
+      --filter 'name=kube-apiserver_kube-apiserver' \
+      --format '{{ "{{.ID}}" }}' \
+    | xargs --no-run-if-empty docker kill
+  args:
+    executable: /bin/bash

core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_common/tasks/extend-kubeadm-config.yml

@@ -0,0 +1,29 @@
+---
+- name: Collect kubeadm-config
+  shell: |
+    kubectl get configmap kubeadm-config \
+      --namespace kube-system \
+      --output jsonpath={{ jsonpath }}
+  vars:
+    jsonpath: >-
+      '{.data.ClusterConfiguration}'
+  environment:
+    KUBECONFIG: /etc/kubernetes/admin.conf
+  args:
+    executable: /bin/bash
+  register: kubeadm_config
+  changed_when: false
+
+- name: Extend kubeadm config
+  set_fact:
+    kubeadm_config: >-
+      {{ original | combine(update, recursive=true) }}
+  vars:
+    original: >-
+      {{ kubeadm_config.stdout | from_yaml }}
+
+- name: Render /etc/kubeadm/kubeadm-config.yml
+  copy:
+    dest: /etc/kubeadm/kubeadm-config.yml
+    content: >-
+      {{ kubeadm_config | to_nice_yaml }}

core/src/epicli/data/common/ansible/playbooks/roles/kubernetes_common/tasks/update-in-cluster-config.yml

@@ -0,0 +1,11 @@
+---
+- name: Update in-cluster configuration
+  shell: |
+    kubeadm init phase upload-config kubeadm \
+      --config /etc/kubeadm/kubeadm-config.yml
+  args:
+    executable: /bin/bash
+  register: upload_config
+  until: upload_config is succeeded
+  retries: 30
+  delay: 10