Deployment for EFK to support job history (#3626)

deployment/k8sPaiLibrary/template/docker-daemon.json.template

@@ -2,7 +2,7 @@
   "log-driver": "json-file",
   "log-opts": {
     "max-size": "2g",
-    "max-file": "1"
+    "max-file": "2"
   },
   "data-root": "{{ hostcofig["docker-data"] }}",
   "runtimes": {

examples/cluster-configuration/services-configuration.yaml

@@ -25,6 +25,9 @@
 #    # HDFS, zookeeper data path on your cluster machine.
 #    data-path: "/datastorage"
 #
+#    # Enable job history feature. Default value is "false"
+#    job-history: "false"
+#
 #    # Enable QoS feature or not. Default value is "true"
 #    qos-switch: "true"
 #

src/cluster/config/cluster.py

@@ -54,6 +54,8 @@ def validation_common(self, common_configuration):
             return False, "cluster-type is not defined or invalid. Please check service-configuration.yaml -> cluster -> common -> cluster-type"
         if "data-path" not in common_configuration:
             return False, "data-path is missing in service-configuration.yaml -> cluster -> common -> data-path"
+        if "job-history" not in common_configuration:
+            return False, "job-history is missing in service-configuration.yaml -> cluster -> common -> job-history"
         if "qos-switch" not in common_configuration:
             return False, "qos-switch is missing in service-configuration.yaml -> cluster -> common -> qos-switch"
         if "az-rdma" not in common_configuration:

src/cluster/config/cluster.yaml

@@ -25,6 +25,8 @@ common:
   # HDFS, zookeeper data path on your cluster machine.
   data-path: "/datastorage"
 
+  job-history: "false"
+
   qos-switch: "true"
 
   az-rdma: "false"

src/elasticsearch/config/elasticsearch.py

@@ -0,0 +1,52 @@
+# Copyright (c) Microsoft Corporation
+# All rights reserved.
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+# documentation files (the "Software"), to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and
+# to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+
+import logging
+import logging.config
+
+
+class Elasticsearch:
+
+    def __init__(self, cluster_configuration, service_configuration, default_service_configuraiton):
+        self.logger = logging.getLogger(__name__)
+
+        self.cluster_configuration = cluster_configuration
+
+
+    def validation_pre(self):
+        for host_config in self.cluster_configuration["machine-list"]:
+            if "pai-master" in host_config and host_config["pai-master"] == "true":
+                return True, None
+
+        return False, "No master node found in machine list"
+
+
+
+    def run(self):
+        com = {"master-ip": None}
+
+        for host_config in self.cluster_configuration["machine-list"]:
+            if "pai-master" in host_config and host_config["pai-master"] == "true":
+                com["master-ip"] = host_config["hostip"]
+                break
+
+        return com
+
+    def validation_post(self, cluster_object_model):
+        return True, None
+

src/elasticsearch/deploy/delete.sh.template

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+# Copyright (c) Microsoft Corporation
+# All rights reserved.
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+# documentation files (the "Software"), to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and
+# to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+{%- if cluster_cfg['cluster']['common']['job-history'] == 'true' %}
+pushd $(dirname "$0") > /dev/null
+
+echo "Call stop.sh to stop elasticsearch first"
+/bin/bash stop.sh || exit $?
+
+echo "Create elasticsearch-delete configmap for deleting data on the host"
+kubectl create configmap elasticsearch-delete --from-file=elasticsearch-delete/ --dry-run -o yaml | kubectl apply --overwrite=true -f - || exit $?
+
+echo "Create cleaner daemon"
+kubectl apply --overwrite=true -f delete.yaml || exit $?
+sleep 5
+
+PYTHONPATH="../../../deployment" python -m  k8sPaiLibrary.monitorTool.check_pod_ready_status -w -k app -v delete-batch-job-elasticsearch || exit $?
+
+echo "Elasticsearch clean job is done"
+echo "Delete elasticsearch cleaner daemon and configmap"
+if kubectl get daemonset | grep -q "delete-batch-job-elasticsearch"; then
+    kubectl delete ds delete-batch-job-elasticsearch || exit $?
+fi
+if kubectl get configmap | grep -q "elasticsearch-delete"; then
+    kubectl delete configmap elasticsearch-delete || exit $?
+fi
+sleep 5
+
+popd > /dev/null
+{%- endif %}

src/elasticsearch/deploy/delete.yaml.template

@@ -0,0 +1,62 @@
+# Copyright (c) Microsoft Corporation
+# All rights reserved.
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+# documentation files (the "Software"), to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and
+# to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: delete-batch-job-elasticsearch
+spec:
+  selector:
+    matchLabels:
+      app: delete-batch-job-elasticsearch
+  template:
+    metadata:
+      labels:
+        app: delete-batch-job-elasticsearch
+    spec:
+      hostNetwork: true
+      hostPID: false
+      containers:
+      - name: cleaning-one-shot
+        image: {{ cluster_cfg["cluster"]["docker-registry"]["prefix"] }}cleaning-image:{{ cluster_cfg["cluster"]["docker-registry"]["tag"] }}
+        imagePullPolicy: Always
+        volumeMounts:
+        - mountPath: /mnt
+          name: data-path
+        - mountPath: /elasticsearch-delete
+          name: elasticsearch-delete-config
+        env:
+        - name: DELETE_CONFIG
+          value: elasticsearch-delete
+        - name: WORKER_CONFIG
+          value: delete-data.sh
+        readinessProbe:
+          exec:
+            command:
+            - cat
+            - /jobstatus/jobok
+          initialDelaySeconds: 5
+          periodSeconds: 3
+      imagePullSecrets:
+      - name: {{ cluster_cfg["cluster"]["docker-registry"]["secret-name"] }}
+      volumes:
+      - name: data-path
+        hostPath:
+          path: {{ cluster_cfg["cluster"]["common"][ "data-path" ] }}
+      - name: elasticsearch-delete-config
+        configMap:
+          name: elasticsearch-delete

src/elasticsearch/deploy/elasticsearch-config.yaml.template

@@ -0,0 +1,47 @@
+# Copyright (c) Microsoft Corporation
+# All rights reserved.
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+# documentation files (the "Software"), to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and
+# to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: elasticsearch-config
+data:
+  elasticsearch.yml: |-
+    cluster:
+      name: elasticsearch-cluster
+      routing.allocation.disk:
+        threshold_enabled: true
+        watermark.low: 80%
+        watermark.high : 85%
+        watermark.flood_stage: 90%
+    discovery.type: single-node
+    xpack.security.enabled: false
+    node:
+      name: elasticsearch-master
+      master: true
+      data: true
+      ingest: true
+    network:
+      host: 0.0.0.0
+  log4j2.properties: |-
+    status = error
+    appender.console.type = Console
+    appender.console.name = console
+    appender.console.layout.type = PatternLayout
+    appender.console.layout.pattern = %d{ISO8601} %p [%t] %c: %m%n
+    rootLogger.level = info
+    rootLogger.appenderRef.console.ref = console

src/elasticsearch/deploy/elasticsearch-delete/delete-data.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+# Copyright (c) Microsoft Corporation
+# All rights reserved.
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+# documentation files (the "Software"), to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and
+# to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+
+echo "Clean elasticsearch data on the disk"
+
+if [ -d "/mnt/elasticsearch/data" ]; then
+
+    rm -rf /mnt/elasticsearch/data
+
+fi

src/elasticsearch/deploy/elasticsearch-service.yaml

@@ -0,0 +1,37 @@
+# Copyright (c) Microsoft Corporation
+# All rights reserved.
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+# documentation files (the "Software"), to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and
+# to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+kind: Service
+apiVersion: v1
+metadata:
+  name: elasticsearch-service
+  labels:
+    app: elasticsearch
+spec:
+  type: NodePort
+  ports:
+    - name: outer-rest
+      port: 9200
+      targetPort: 9200
+      nodePort: 30100
+      protocol: TCP
+    - name: inter-node
+      port: 9300
+      targetPort: 9300
+      protocol: TCP
+  selector:
+    app: elasticsearch

src/elasticsearch/deploy/elasticsearch.yaml.template

@@ -0,0 +1,76 @@
+# Copyright (c) Microsoft Corporation
+# All rights reserved.
+#
+# MIT License
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+# documentation files (the "Software"), to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and
+# to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
+# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+kind: StatefulSet
+apiVersion: apps/v1
+metadata:
+  name: elasticsearch-sts
+  labels:
+    app: elasticsearch
+spec:
+  replicas: 1
+  serviceName: elasticsearch
+  selector:
+    matchLabels:
+      app: elasticsearch
+  template:
+    metadata:
+      labels:
+        app: elasticsearch
+    spec:
+      containers:
+      - name: elasticsearch
+        image: elasticsearch:7.4.0
+        ports:
+        - name: outer-rest
+          containerPort: 9200
+          protocol: TCP
+        - name: inter-node
+          containerPort: 9300
+          protocol: TCP
+        env:
+        - name: "ES_JAVA_OPTS"
+          value: "-Xms1024m -Xmx1024m"
+        - name: TAKE_FILE_OWNERSHIP
+          value: "true"
+        volumeMounts:
+        - name: elasticsearch-config
+          mountPath: /usr/share/elasticsearch/config/elasticsearch.yml
+          subPath: elasticsearch.yml
+          readOnly: true
+        - name: elasticsearch-config
+          mountPath: /usr/share/elasticsearch/config/log4j2.properties
+          subPath: log4j2.properties
+        - name: elasticsearch-data
+          mountPath: /usr/share/elasticsearch/data
+      volumes:
+      - name: elasticsearch-config
+        configMap:
+           name: elasticsearch-config
+      - name:  elasticsearch-data
+        hostPath:
+          path: {{ cluster_cfg["cluster"]["common"][ "data-path" ] }}/elasticsearch/data
+          type: DirectoryOrCreate
+      # Elasticsearch requires vm.max_map_count to be at least 262144.
+      # If your OS already sets up this number to a higher value, feel free
+      # to remove this init container.
+      initContainers:
+      - image: alpine:3.6
+        command: ["/sbin/sysctl", "-w", "vm.max_map_count=262144"]
+        name: elasticsearch-logging-init
+        securityContext:
+          privileged: true