Merge pull request #169 from leeoo/dev2.0

Add MinIO integration support

cloudeon-stack/EDP-2.0.0/minio/alert-rule.yaml

@@ -0,0 +1,13 @@
+rules:
+    - alert: MinIO存活数检测
+      promql: sum(up{job="metrics-minio"})<#{{${serviceRoles['MINIO_SERVER']?size}}}#
+      alertLevel: exception
+      serviceRoleName: MINIO_SERVER
+      alertAdvice: MinIO发生宕机，请检查日志或执行重启
+      alertInfo: "MinIO Server发生宕机，当前存活实例数为：{{$value}}"
+    - alert: MinIO Server进程存活
+      promql: up{job="metrics-minio"} != 1
+      alertLevel: exception
+      serviceRoleName: MINIO_SERVER
+      alertAdvice: MinIO Server宕机，请重新启动
+      alertInfo: "{{ $labels.job }}的{{ $labels.instance }}实例产生告警"

cloudeon-stack/EDP-2.0.0/minio/docker/Dockerfile

@@ -0,0 +1,30 @@
+FROM rockylinux:9
+
+LABEL baseos=rockylinux9
+
+ENV TZ=Asia/Shanghai  \
+    FREEMARKER_GENERATOR_CLI_HOME=/opt/freemarker-generator-cli
+ENV PATH=$PATH:$FREEMARKER_GENERATOR_CLI_HOME/bin
+ENV MINIO_HOME=/opt/minio \
+    MINIO_VERSION=2021-04-22T15-44-28Z.hotfix.2cc0aaed8
+ENV PATH=${PATH}:${MINIO_HOME}/bin \
+    MINIOBINDIR=${MINIO_HOME}/bin
+ENV JAVA_HOME=/opt/jdk1.8.0_331
+ENV PATH=$JAVA_HOME/bin:$PATH \
+    LD_LIBRARY_PATH=$JAVA_HOME/lib/amd64/jli:$JAVA_HOME/lib/amd64
+
+ADD freemarker-generator-cli-0.2.0-SNAPSHOT-app.tar.gz jdk-8u331-linux-x64.tar.gz /opt/
+
+RUN mkdir -p $MINIO_HOME/bin \
+    && mkdir $MINIO_HOME/conf \
+    && mkdir /data \
+    && curl -o $MINIO_HOME/bin/minio https://dl.min.io/server/minio/hotfixes/linux-amd64/archive/minio.RELEASE.${MINIO_VERSION} \
+#     && mv minio.RELEASE.${MINIO_VERSION} $MINIO_HOME/bin/minio \
+    && chmod +x $MINIO_HOME/bin/minio \
+    && mv /opt/freemarker-generator-cli-* /opt/freemarker-generator-cli \
+    && ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime && echo 'Asia/Shanghai' >/etc/timezone \
+    # 安装 cron、nc 等命令
+    && dnf install -y cronie \
+    && dnf install -y nmap \
+    && dnf clean all
+WORKDIR $MINIO_HOME

cloudeon-stack/EDP-2.0.0/minio/docker/build.sh

@@ -0,0 +1,2 @@
+docker build -t registry.cn-guangzhou.aliyuncs.com/bigdata200/minio:2021-04-22T15-44-28Z .
+docker push  registry.cn-guangzhou.aliyuncs.com/bigdata200/minio:2021-04-22T15-44-28Z

cloudeon-stack/EDP-2.0.0/minio/k8s-render/metrics-service.yaml.ftl

@@ -0,0 +1,32 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: minio-service-monitor
+  labels:
+    team: frontend
+spec:
+  selector:
+    matchLabels:
+      sname: ${serviceFullName}
+  endpoints:
+  - port: metrics
+    path: /minio/prometheus/metrics
+  - port: metrics
+    path: /minio/v2/metrics/cluster
+  - port: metrics
+    path: /minio/v2/metrics/node
+---
+kind: Service
+apiVersion: v1
+metadata:
+  name: metrics-minio
+  labels:
+    sname: ${serviceFullName}
+    roleFullName: minio-server
+spec:
+  selector:
+    sname: ${serviceFullName}
+    roleFullName: minio-server
+  ports:
+  - name: metrics
+    port: ${conf['minio.metrics.port']}

cloudeon-stack/EDP-2.0.0/minio/k8s/minio-server.yaml.ftl

@@ -0,0 +1,208 @@
+<#if conf["data.path.list"]??&& conf["data.path.list"]?trim?has_content>
+    <#assign primeDataPathList=conf["data.path.list"]?trim?split(",")>
+<#else >
+    <#assign primeDataPathList=[conf['global.persistence.basePath']]>
+</#if>
+<#assign dataPathList = []>
+<#list primeDataPathList as dataPath>
+    <#if dataPath?ends_with("/")>
+      <#assign dataPathList = dataPathList + [dataPath+ roleFullName]>
+    <#else>
+      <#assign dataPathList = dataPathList + [dataPath+"/"+ roleFullName]>
+    </#if>
+</#list>
+---
+apiVersion: "apps/v1"
+kind: "Deployment"
+metadata:
+  labels:
+    name: "${roleServiceFullName}"
+    sname: "${serviceFullName}"
+    roleFullName: "${roleFullName}"
+  name: "${roleServiceFullName}"
+spec:
+  replicas: ${roleNodeCnt}
+  selector:
+    matchLabels:
+      app: "${roleServiceFullName}"
+      sname: "${serviceFullName}"
+      roleFullName: "${roleFullName}"
+  strategy:
+    type: "RollingUpdate"
+    rollingUpdate:
+      maxSurge: 0
+      maxUnavailable: 1
+  minReadySeconds: 5
+  revisionHistoryLimit: 10
+  template:
+    metadata:
+      labels:
+        name: "${roleServiceFullName}"
+        sname: "${serviceFullName}"
+        roleFullName: "${roleFullName}"
+        app: "${roleServiceFullName}"
+        podConflictName: "${roleServiceFullName}"
+        inject-filebeat: "true"
+    spec:
+      affinity:
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+          - labelSelector:
+              matchLabels:
+                name: "${roleServiceFullName}"
+                podConflictName: "${roleServiceFullName}"
+            topologyKey: "kubernetes.io/hostname"
+      hostPID: false
+      hostNetwork: true
+      nodeSelector:
+        ${roleServiceFullName}: "true"
+      terminationGracePeriodSeconds: 30
+      containers:
+<#if dependencies.FILEBEAT??>
+      - name: filebeat
+        image: "${dependencies.FILEBEAT.conf['serverImage']}"
+        imagePullPolicy: "${conf['global.imagePullPolicy']}"
+    # 设置root用户运行，避免写入filebeat.registry失败
+        securityContext:
+          runAsUser: 0
+        env:
+          - name: NODE_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: spec.nodeName
+          - name: POD_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.name
+          - name: POD_IP
+            valueFrom:
+              fieldRef:
+                fieldPath: status.podIP
+    <#--禁用filebeat 检测，避免因filebeat状态异常影响安装-->
+    <#--    readinessProbe:-->
+    <#--      exec:-->
+    <#--        command:-->
+    <#--        - sh-->
+    <#--        - -c-->
+    <#--        - |-->
+    <#--          #!/usr/bin/env bash -e-->
+    <#--          filebeat test output-->
+        resources:
+          limits:
+            cpu: "1"
+            memory: 200Mi
+          requests:
+            cpu: 10m
+            memory: 100Mi
+        volumeMounts:
+        - mountPath: /usr/share/filebeat/filebeat.yml
+          name: filebeat-config
+          subPath: filebeat.yml
+        - mountPath: "/workspace/filebeat"
+          name: "workspace"
+          subPath: filebeat
+        - mountPath: "/workspace/logs"
+          name: "workspace"
+          subPath: logs
+</#if>
+      - image: "${conf['serverImage']}"
+        imagePullPolicy: "${conf['global.imagePullPolicy']}"
+        name: "${roleServiceFullName}"
+        command: ["/bin/bash","-c"]
+        args:
+          - |
+            /bin/bash /opt/global/bootstrap.sh && \
+            /bin/bash /opt/service-common/bootstrap.sh;
+        readinessProbe:
+          exec:
+            command:
+            - "/bin/bash"
+            - "/opt/service-common/readiness.sh"
+          failureThreshold: 3
+          initialDelaySeconds: 3
+          periodSeconds: 30
+          successThreshold: 1
+          timeoutSeconds: 15
+        resources:
+          requests:
+            memory: "${conf['minio.container.request.memory']}Mi"
+            cpu: "${conf['minio.container.request.cpu']}"
+          limits:
+            memory: "${conf['minio.container.limit.memory']}Mi"
+            cpu: "${conf['minio.container.limit.cpu']}"
+        env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        - name: MEM_LIMIT
+          valueFrom:
+            resourceFieldRef:
+              resource: limits.memory
+        - name: RENDER_TPL_DIR
+          value: "/opt/service-render"
+        - name: RENDER_MODEL
+          value: "/opt/service-common/values.json"
+        - name: MINIO_SERVER_PORT
+          value: "${conf["minio.server.port"]}"
+        - name: MINIO_METRICS_PORT
+          value: "${conf["minio.metrics.port"]}"
+        volumeMounts:
+        - mountPath: "/etc/localtime"
+          name: "timezone"
+        - name: global-service-common
+          mountPath: /opt/global/bootstrap.sh
+          subPath: bootstrap.sh
+        - name: global-render-config
+          mountPath: /opt/global/10.render
+        - name: global-usersync-config
+          mountPath: /opt/global/20.usersync
+        - name: global-copy-filebeat-config
+          mountPath: /opt/global/30.copy-filebeat-config
+        - name: service-render
+          mountPath: /opt/service-render
+        - name: service-common
+          mountPath: /opt/service-common
+        - mountPath: "/workspace"
+          name: "workspace"
+<#list dataPathList as dataPath>
+        - name: local-data-${dataPath?index+1}
+          mountPath: /data/${dataPath?index+1}
+</#list>
+      volumes:
+      - hostPath:
+          path: "/etc/localtime"
+        name: "timezone"
+      - name: global-service-common
+        configMap:
+          name: global-service-common
+      - name: global-render-config
+        configMap:
+          name: global-render-config
+      - name: global-usersync-config
+        configMap:
+          name: global-usersync-config
+      - name: global-copy-filebeat-config
+        configMap:
+          name: global-copy-filebeat-config
+      - name: service-render
+        configMap:
+          name: minio-service-render
+      - name: service-common
+        configMap:
+          name: minio-service-common
+      - name: "workspace"
+        hostPath:
+          type: DirectoryOrCreate
+          path: "${dataPathList[0]}/workspace"
+<#list dataPathList as dataPath>
+      - name: local-data-${dataPath?index+1}
+        hostPath:
+          path: ${dataPath}/data
+          type: DirectoryOrCreate
+</#list>
+<#if dependencies.FILEBEAT??>
+      - configMap:
+          name: filebeat-common-config
+        name: filebeat-config
+</#if>

cloudeon-stack/EDP-2.0.0/minio/kube-prometheus-render/alert-rule.yaml.ftl

@@ -0,0 +1,33 @@
+<#macro executeDynamicCode userInput>
+    <#assign startIdx = userInput?index_of('#{{')>
+    <#assign endIdx = userInput?last_index_of('}}#')>
+    <#if startIdx gt 0 && endIdx gt 0>
+        <#assign dynamicCode = userInput?substring(startIdx + 3, endIdx)>
+        ${userInput?substring(0,startIdx)}<@dynamicCode?interpret />${userInput?substring(endIdx+3)}<#t>
+    <#else >
+        ${userInput}
+    </#if><#t>
+</#macro>
+<#if alertRules ?? && alertRules?size gt 0>
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: ${serviceFullName}
+spec:
+  groups:
+    - name: ${serviceFullName}
+      rules:
+      <#list alertRules as rule>
+      -  alert: ${rule.ruleName}
+         expr: <@executeDynamicCode rule.promql />
+         labels:
+             receiver: "webhook"
+             alertLevel: "${rule.alertLevel}"
+             clusterId: "${rule.clusterId}"
+             serviceRoleName: "${rule.stackRoleName}"
+             serviceName: "${rule.stackServiceName}"
+         annotations:
+             alertAdvice: "${rule.alertAdvice}"
+             alertInfo:  "${rule.alertInfo}"
+      </#list>
+</#if>