Add data retention methods for in-memory Clickhouse deployment

Signed-off-by: Yanjun Zhou <zhouya@vmware.com>

.github/workflows/build.yml

@@ -162,3 +162,20 @@ jobs:
       run: |
         echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
         docker push antrea/flow-aggregator:latest
+
+  build-flow-visibility-clickhouse-monitor:
+    needs: check-changes
+    if: ${{ needs.check-changes.outputs.has_changes == 'yes' || github.event_name == 'push' }}
+    runs-on: [ubuntu-latest]
+    steps:
+    - uses: actions/checkout@v2
+    - name: Build flow-visibility-clickhouse-monitor Docker image
+      run: make flow-visibility-clickhouse-monitor
+    - name: Push flow-visibility-clickhouse-monitor Docker image to registry
+      if: ${{ github.repository == 'antrea-io/antrea' && github.event_name == 'push' && github.ref == 'refs/heads/main' }}
+      env:
+        DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+        DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+      run: |
+        echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
+        docker push antrea/flow-visibility-clickhouse-monitor:latest

.github/workflows/build_tag.yml

@@ -104,3 +104,18 @@ jobs:
         make flow-aggregator-image
         echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
         docker push antrea/flow-aggregator:"${VERSION}"
+
+  build-flow-visibility-clickhouse-monitor:
+    runs-on: [ubuntu-latest]
+    needs: get-version
+    steps:
+    - uses: actions/checkout@v2
+    - name: Build flow-visibility-clickhouse-monitor Docker image and push to registry
+      env:
+        DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+        DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+        VERSION: ${{ needs.get-version.outputs.version }}
+      run: |
+        make flow-visibility-clickhouse-monitor
+        echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
+        docker push antrea/flow-visibility-clickhouse-monitor:"${VERSION}"

Makefile

@@ -371,6 +371,14 @@ endif
 	docker tag antrea/antrea-mc-controller:$(DOCKER_IMG_VERSION) projects.registry.vmware.com/antrea/antrea-mc-controller
 	docker tag antrea/antrea-mc-controller:$(DOCKER_IMG_VERSION) projects.registry.vmware.com/antrea/antrea-mc-controller:$(DOCKER_IMG_VERSION)
 
+.PHONY: flow-visibility-clickhouse-monitor
+flow-visibility-clickhouse-monitor:
+	@echo "===> Building antrea/flow-visibility-clickhouse-monitor Docker image <==="
+	docker build --pull -t antrea/flow-visibility-clickhouse-monitor:$(DOCKER_IMG_VERSION) -f build/images/Dockerfile.clickhouse.monitor.ubuntu $(DOCKER_BUILD_ARGS) .
+	docker tag antrea/flow-visibility-clickhouse-monitor:$(DOCKER_IMG_VERSION) antrea/flow-visibility-clickhouse-monitor
+	docker tag antrea/flow-visibility-clickhouse-monitor:$(DOCKER_IMG_VERSION) projects.registry.vmware.com/antrea/flow-visibility-clickhouse-monitor
+	docker tag antrea/flow-visibility-clickhouse-monitor:$(DOCKER_IMG_VERSION) projects.registry.vmware.com/antrea/flow-visibility-clickhouse-monitor:$(DOCKER_IMG_VERSION)
+
 .PHONY: flow-aggregator-image
 flow-aggregator-image:
 	@echo "===> Building antrea/flow-aggregator Docker image <==="

build/images/Dockerfile.clickhouse.monitor.ubuntu

@@ -0,0 +1,19 @@
+ARG GO_VERSION
+FROM golang:${GO_VERSION} as clickhouse-monitor-build
+
+COPY . /antrea
+WORKDIR /antrea/plugins/flow-visibility/clickhouse-monitor
+
+# Statically links clickhouse-monitor-plugin binary to make sure it can run in scratch.
+RUN CGO_ENABLED=0 make clickhouse-monitor-plugin
+
+FROM scratch
+
+LABEL maintainer="Antrea <projectantrea-dev@googlegroups.com>"
+LABEL description="A docker image to deploy the clickhouse monitor plugin."
+
+ENV USER root
+
+COPY --from=clickhouse-monitor-build /antrea/plugins/flow-visibility/clickhouse-monitor/* /
+
+ENTRYPOINT ["/clickhouse-monitor"]

build/yamls/flow-visibility/flow-visibility.yaml

@@ -204,7 +204,7 @@ data:
   create_table.sh: |
     #!/bin/bash
     set -e
-    clickhouse client -n <<-EOSQL
+    clickhouse client -n -h 127.0.0.1<<-EOSQL
       CREATE TABLE IF NOT EXISTS flows (
           timeInserted DateTime,
           flowStartSeconds DateTime,
@@ -256,7 +256,10 @@ data:
           reverseThroughputFromDestinationNode UInt64,
           trusted Bool
       ) engine=MergeTree
-      ORDER BY (timeInserted, flowEndSeconds);
+      ORDER BY (timeInserted, flowEndSeconds)
+      TTL timeInserted + INTERVAL 10 MINUTE
+      SETTINGS merge_with_ttl_timeout = 600;
+
       CREATE MATERIALIZED VIEW flows_pod_view
       ENGINE = SummingMergeTree
       ORDER BY (
@@ -400,11 +403,108 @@ spec:
         spec:
           containers:
             - name: clickhouse
-              image: yandex/clickhouse-server:20.7
+              image: yandex/clickhouse-server:21.11
               volumeMounts:
                 - name: clickhouse-configmap-volume
                   mountPath: /docker-entrypoint-initdb.d
+                - name: clickhouse-storage-volume
+                  mountPath: /var/lib/clickhouse
           volumes:
             - name: clickhouse-configmap-volume
               configMap:
-                name: clickhouse-mounted-configmap
+                name: clickhouse-mounted-configmap
+            - name: clickhouse-storage-volume
+              emptyDir:
+                medium: Memory
+                sizeLimit: 2Gi
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    app: flow-visibility
+  name: clickhouse-monitor
+  namespace: flow-visibility
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  labels:
+    app: flow-visibility
+  name: clickhouse-monitor-role
+  namespace: flow-visibility
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+    verbs:
+      - list
+  - apiGroups:
+      - ""
+    resources:
+      - pods/log
+    verbs:
+      - get
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  labels:
+    app: flow-visibility
+  name: clickhouse-monitor-role-binding
+  namespace: flow-visibility
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: clickhouse-monitor-role
+subjects:
+  - kind: ServiceAccount
+    name: clickhouse-monitor
+    namespace: flow-visibility
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  labels:
+    app: clickhouse-monitor
+  name: clickhouse-monitor
+  namespace: flow-visibility
+spec:
+  schedule: "* * * * *"
+  successfulJobsHistoryLimit: 1
+  failedJobsHistoryLimit: 1
+  jobTemplate:
+    spec:
+      template:
+        metadata:
+          labels:
+            app: clickhouse-monitor
+        spec:
+          serviceAccountName: clickhouse-monitor
+          containers:
+          - name: clickhouse-monitor
+            image: projects.registry.vmware.com/antrea/flow-visibility-clickhouse-monitor:latest
+            imagePullPolicy: IfNotPresent
+            env:
+              - name: CH_USERNAME
+                valueFrom:
+                  secretKeyRef: 
+                    name: clickhouse-secret
+                    key: username
+              - name: CH_PASSWORD
+                valueFrom:
+                  secretKeyRef:
+                    name: clickhouse-secret
+                    key: password
+              - name: SVC_HOST
+                value: "clickhouse-clickhouse.flow-visibility.svc.cluster.local"
+              - name: SVC_PORT
+                value: "9000"
+              - name: TABLE_NAME
+                value: "default.flows"
+              - name: NAMESPACE
+                value: "flow-visibility"
+              - name: MONITOR_LABEL
+                value: "app=clickhouse-monitor"
+          restartPolicy: OnFailure

plugins/flow-visibility/clickhouse-monitor/Makefile

@@ -0,0 +1,10 @@
+SHELL                            := /bin/bash
+GO                               ?= go
+LDFLAGS                          :=
+GOFLAGS                          :=
+BINDIR                           ?= $(CURDIR)/bin
+
+.PHONY: clickhouse-monitor-plugin
+clickhouse-monitor-plugin:
+	@mkdir -p $(BINDIR)
+	GOOS=linux $(GO) build -o $(BINDIR) $(GOFLAGS) -ldflags '$(LDFLAGS)' antrea.io/antrea/plugins/flow-visibility/clickhouse-monitor

plugins/flow-visibility/clickhouse-monitor/go.mod

@@ -0,0 +1,41 @@
+module antrea.io/antrea/plugins/flow-visibility/clickhouse-monitor
+
+go 1.17
+
+require (
+	github.com/ClickHouse/clickhouse-go v1.5.1
+	k8s.io/api v0.23.1
+	k8s.io/apimachinery v0.23.1
+	k8s.io/client-go v0.23.1
+	k8s.io/klog/v2 v2.30.0
+)
+
+require (
+	github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/go-logr/logr v1.2.0 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/google/go-cmp v0.5.5 // indirect
+	github.com/google/gofuzz v1.1.0 // indirect
+	github.com/googleapis/gnostic v0.5.5 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	golang.org/x/net v0.0.0-20211209124913-491a49abca63 // indirect
+	golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f // indirect
+	golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e // indirect
+	golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b // indirect
+	golang.org/x/text v0.3.7 // indirect
+	golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/protobuf v1.27.1 // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
+	k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 // indirect
+	k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b // indirect
+	sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.1.2 // indirect
+	sigs.k8s.io/yaml v1.2.0 // indirect
+)