Add Grafana dashboards and update doc

1. Add Grafana dashboards
2. Change deployment files arrangements and manifest generation and maintenance
3. Add documentation for flow-visibility solution

Signed-off-by: heanlan <hanlan@vmware.com>

.github/workflows/upload_release_assets.yml

@@ -191,6 +191,15 @@ jobs:
         asset_path: ./assets/flow-aggregator.yml
         asset_name: flow-aggregator.yml
         asset_content_type: application/octet-stream
+    - name: Upload flow-visibility.yaml
+      uses: actions/upload-release-asset@v1
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        upload_url: ${{ github.event.release.upload_url }}
+        asset_path: ./assets/flow-visibility.yaml
+        asset_name: flow-visibility.yaml
+        asset_content_type: application/octet-stream
     - name: Upload antrea-multicluster-leader-global.yml
       uses: actions/upload-release-asset@v1
       env:

Makefile

@@ -347,6 +347,7 @@ manifest:
 	$(CURDIR)/hack/generate-manifest-octant.sh --mode dev > build/yamls/antrea-octant.yml
 	$(CURDIR)/hack/generate-manifest-windows.sh --mode dev > build/yamls/antrea-windows.yml
 	$(CURDIR)/hack/generate-manifest-flow-aggregator.sh --mode dev > build/yamls/flow-aggregator.yml
+	$(CURDIR)/hack/generate-manifest-flow-visibility.sh > build/yamls/flow-visibility.yaml
 
 .PHONY: manifest-scale
 manifest-scale:

build/yamls/flow-visibility/base/clickhouse.yaml

@@ -0,0 +1,30 @@
+apiVersion: "clickhouse.altinity.com/v1"
+kind: "ClickHouseInstallation"
+metadata:
+  name: clickhouse
+  labels:
+    app: clickhouse
+spec:
+  configuration:
+    clusters:
+      - name: "clickhouse"
+        layout:
+          shardsCount: 1
+          replicasCount: 1
+  defaults:
+    templates:
+      podTemplate: pod-template
+  templates:
+    podTemplates:
+      - name: pod-template
+        spec:
+          containers:
+            - name: clickhouse
+              image: yandex/clickhouse-server:21.11
+              volumeMounts:
+                - name: clickhouse-configmap-volume
+                  mountPath: /docker-entrypoint-initdb.d
+          volumes:
+            - name: clickhouse-configmap-volume
+              configMap:
+                name: $(CH_CONF)

build/yamls/flow-visibility/base/grafana.yaml

@@ -0,0 +1,182 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: flow-visibility
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: grafana
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  labels:
+    app: flow-visibility
+  name: grafana-role
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - services
+    verbs:
+      - get
+      - list
+      - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  labels:
+    app: flow-visibility
+  name: grafana-role-binding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: grafana-role
+subjects:
+  - kind: ServiceAccount
+    name: grafana
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: grafana-storage
+provisioner: kubernetes.io/no-provisioner
+volumeBindingMode: WaitForFirstConsumer
+reclaimPolicy: Delete
+allowVolumeExpansion: True
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: grafana-pvc
+spec:
+  storageClassName: grafana-storage
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 1Gi
+---
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: grafana-pv
+spec:
+  storageClassName: grafana-storage
+  capacity:
+    storage: 2Gi
+  accessModes:
+    - ReadWriteOnce
+  hostPath:
+    path: "/data/grafana"
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: clickhouse-secret
+type: Opaque
+stringData:
+  username: clickhouse_operator
+  password: clickhouse_operator_password
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: grafana
+  name: grafana
+spec:
+  selector:
+    matchLabels:
+      app: grafana
+  template:
+    metadata:
+      labels:
+        app: grafana
+    spec:
+      serviceAccountName: grafana
+      securityContext:
+        fsGroup: 472
+        supplementalGroups:
+          - 0
+      containers:
+        - name: grafana
+          image: grafana/grafana:8.3.3
+          imagePullPolicy: IfNotPresent
+          env:
+            - name: GF_INSTALL_PLUGINS
+              value: "https://downloads.antrea.io/artifacts/grafana-custom-plugins/grafana-sankey-plugin-1.0.0.zip;antreaflowvisibility-grafana-sankey-plugin,grafana-clickhouse-datasource 0.12.0"
+            - name: CH_USERNAME
+              valueFrom:
+                secretKeyRef: 
+                  name: clickhouse-secret
+                  key: username
+            - name: CH_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: clickhouse-secret
+                  key: password
+          ports:
+            - containerPort: 3000
+              name: http-grafana
+              protocol: TCP
+          readinessProbe:
+            failureThreshold: 3
+            httpGet:
+              path: /robots.txt
+              port: 3000
+              scheme: HTTP
+            initialDelaySeconds: 10
+            periodSeconds: 30
+            successThreshold: 1
+            timeoutSeconds: 2
+          livenessProbe:
+            failureThreshold: 3
+            initialDelaySeconds: 30
+            periodSeconds: 10
+            successThreshold: 1
+            tcpSocket:
+              port: 3000
+            timeoutSeconds: 1
+          resources:
+            requests:
+              cpu: 250m
+              memory: 750Mi
+          volumeMounts:
+            - mountPath: /data
+              name: grafana-pv
+            - mountPath: /etc/grafana/provisioning/datasources
+              name: grafana-datasource-provider
+            - mountPath: /etc/grafana/provisioning/dashboards
+              name: grafana-dashboard-provider
+            - mountPath: /var/lib/grafana/dashboards
+              name: grafana-dashboard-config
+      volumes:
+        - name: grafana-pv
+          persistentVolumeClaim:
+            claimName: grafana-pvc
+        - name: grafana-datasource-provider
+          configMap:
+            name: grafana-datasource-provider
+        - name: grafana-dashboard-provider
+          configMap:
+            name: grafana-dashboard-provider
+        - name: grafana-dashboard-config
+          configMap:
+            name: grafana-dashboard-config
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: grafana
+spec:
+  ports:
+    - port: 3000
+      protocol: TCP
+      targetPort: http-grafana
+  selector:
+    app: grafana
+  sessionAffinity: None
+  type: LoadBalancer

build/yamls/flow-visibility/base/kustomization.yaml

@@ -0,0 +1,41 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: flow-visibility
+
+resources:
+- clickhouse.yaml
+- grafana.yaml
+
+configMapGenerator:
+- name: grafana-datasource-provider
+  files:
+  - provisioning/datasources/datasource_provider.yaml
+- name: grafana-dashboard-provider
+  files:
+  - provisioning/dashboards/dashboard_provider.yaml
+- name: clickhouse-mounted-configmap
+  namespace: flow-visibility
+  files:
+  - provisioning/datasources/create_table.sh
+- name: grafana-dashboard-config
+  files:
+   - provisioning/dashboards/flow_records_dashboard.json
+   - provisioning/dashboards/pod_to_pod_dashboard.json
+   - provisioning/dashboards/pod_to_service_dashboard.json
+   - provisioning/dashboards/pod_to_external_dashboard.json
+   - provisioning/dashboards/node_to_node_dashboard.json
+   - provisioning/dashboards/networkpolicy_allow_dashboard.json
+
+# CH_CONF exports the value in `metadata.name` from `ConfigMap` named `clickhouse-mounted-configmap`,
+# which is used for inserting the value to a CRD for an object of kind `ClickHouseInstallation`
+vars:
+- name: CH_CONF
+  objref:
+    kind: ConfigMap
+    name: clickhouse-mounted-configmap
+    apiVersion: v1
+  fieldref:
+    fieldpath: metadata.name
+
+configurations:
+- kustomize-config.yaml

build/yamls/flow-visibility/base/kustomize-config.yaml

@@ -0,0 +1,5 @@
+# This file defines the resource and the field that Kustomize need to look up for
+# the value of var `CH_CONF`
+varReference:
+- path: spec/templates/podTemplates/spec/volumes/configMap/name
+  kind: ClickHouseInstallation

build/yamls/flow-visibility/base/provisioning/dashboards/dashboard_provider.yaml

@@ -0,0 +1,8 @@
+apiVersion: 1
+providers:
+  - name: grafana-dashboards
+    folder: ''
+    type: file
+    allowUiUpdates: true
+    options:
+      path: /var/lib/grafana/dashboards