Add data retention methods for in-memory Clickhouse deployment

Signed-off-by: Yanjun Zhou <zhouya@vmware.com>
antrea-io · Mar 1, 2022 · bc22847 · bc22847
1 parent f1778f0
commit bc22847
Show file tree

Hide file tree

Showing 16 changed files with 1,329 additions and 14 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -182,3 +182,20 @@ jobs:
       run: |
         echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
         docker push antrea/flow-aggregator:latest
+
+  build-flow-visibility-clickhouse-monitor:
+    needs: check-changes
+    if: ${{ needs.check-changes.outputs.has_changes == 'yes' || github.event_name == 'push' }}
+    runs-on: [ubuntu-latest]
+    steps:
+    - uses: actions/checkout@v2
+    - name: Build flow-visibility-clickhouse-monitor Docker image
+      run: make flow-visibility-clickhouse-monitor
+    - name: Push flow-visibility-clickhouse-monitor Docker image to registry
+      if: ${{ github.repository == 'antrea-io/antrea' && github.event_name == 'push' && github.ref == 'refs/heads/main' }}
+      env:
+        DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+        DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+      run: |
+        echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
+        docker push antrea/flow-visibility-clickhouse-monitor:latest
diff --git a/.github/workflows/build_tag.yml b/.github/workflows/build_tag.yml
@@ -119,3 +119,18 @@ jobs:
         make flow-aggregator-image
         echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
         docker push antrea/flow-aggregator:"${VERSION}"
+
+  build-flow-visibility-clickhouse-monitor:
+    runs-on: [ubuntu-latest]
+    needs: get-version
+    steps:
+    - uses: actions/checkout@v2
+    - name: Build flow-visibility-clickhouse-monitor Docker image and push to registry
+      env:
+        DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+        DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+        VERSION: ${{ needs.get-version.outputs.version }}
+      run: |
+        make flow-visibility-clickhouse-monitor
+        echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
+        docker push antrea/flow-visibility-clickhouse-monitor:"${VERSION}"
diff --git a/Makefile b/Makefile
@@ -355,7 +355,7 @@ manifest:
 	$(CURDIR)/hack/generate-manifest-octant.sh --mode dev > build/yamls/antrea-octant.yml
 	$(CURDIR)/hack/generate-manifest-windows.sh --mode dev > build/yamls/antrea-windows.yml
 	$(CURDIR)/hack/generate-manifest-flow-aggregator.sh --mode dev > build/yamls/flow-aggregator.yml
-	$(CURDIR)/hack/generate-manifest-flow-visibility.sh > build/yamls/flow-visibility.yaml
+	$(CURDIR)/hack/generate-manifest-flow-visibility.sh --mode dev > build/yamls/flow-visibility.yaml
 
 .PHONY: manifest-scale
 manifest-scale:
@@ -388,6 +388,14 @@ endif
 	docker tag antrea/antrea-mc-controller:$(DOCKER_IMG_VERSION) projects.registry.vmware.com/antrea/antrea-mc-controller
 	docker tag antrea/antrea-mc-controller:$(DOCKER_IMG_VERSION) projects.registry.vmware.com/antrea/antrea-mc-controller:$(DOCKER_IMG_VERSION)
 
+.PHONY: flow-visibility-clickhouse-monitor
+flow-visibility-clickhouse-monitor:
+	@echo "===> Building antrea/flow-visibility-clickhouse-monitor Docker image <==="
+	docker build --pull -t antrea/flow-visibility-clickhouse-monitor:$(DOCKER_IMG_VERSION) -f build/images/Dockerfile.clickhouse.monitor.ubuntu $(DOCKER_BUILD_ARGS) .
+	docker tag antrea/flow-visibility-clickhouse-monitor:$(DOCKER_IMG_VERSION) antrea/flow-visibility-clickhouse-monitor
+	docker tag antrea/flow-visibility-clickhouse-monitor:$(DOCKER_IMG_VERSION) projects.registry.vmware.com/antrea/flow-visibility-clickhouse-monitor
+	docker tag antrea/flow-visibility-clickhouse-monitor:$(DOCKER_IMG_VERSION) projects.registry.vmware.com/antrea/flow-visibility-clickhouse-monitor:$(DOCKER_IMG_VERSION)
+
 .PHONY: flow-aggregator-image
 flow-aggregator-image:
 	@echo "===> Building antrea/flow-aggregator Docker image <==="

diff --git a/build/images/Dockerfile.clickhouse.monitor.ubuntu b/build/images/Dockerfile.clickhouse.monitor.ubuntu
@@ -0,0 +1,19 @@
+ARG GO_VERSION
+FROM golang:${GO_VERSION} as clickhouse-monitor-build
+
+COPY . /antrea
+WORKDIR /antrea/plugins/flow-visibility/clickhouse-monitor
+
+# Statically links clickhouse-monitor-plugin binary to make sure it can run in scratch.
+RUN CGO_ENABLED=0 make clickhouse-monitor-plugin
+
+FROM scratch
+
+LABEL maintainer="Antrea <projectantrea-dev@googlegroups.com>"
+LABEL description="A docker image to deploy the clickhouse monitor plugin."
+
+ENV USER root
+
+COPY --from=clickhouse-monitor-build /antrea/plugins/flow-visibility/clickhouse-monitor/* /
+
+ENTRYPOINT ["/clickhouse-monitor"]
diff --git a/build/yamls/flow-visibility.yaml b/build/yamls/flow-visibility.yaml
diff --git a/build/yamls/flow-visibility/base/clickhouse.yaml b/build/yamls/flow-visibility/base/clickhouse.yaml
@@ -24,7 +24,100 @@ spec:
               volumeMounts:
                 - name: clickhouse-configmap-volume
                   mountPath: /docker-entrypoint-initdb.d
+                - name: clickhouse-storage-volume
+                  mountPath: /var/lib/clickhouse
           volumes:
             - name: clickhouse-configmap-volume
               configMap:
                 name: $(CH_CONF)
+            - name: clickhouse-storage-volume
+              emptyDir:
+                medium: Memory
+                sizeLimit: 2Gi
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    app: flow-visibility
+  name: clickhouse-monitor
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  labels:
+    app: flow-visibility
+  name: clickhouse-monitor-role
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+    verbs:
+      - list
+  - apiGroups:
+      - ""
+    resources:
+      - pods/log
+    verbs:
+      - get
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  labels:
+    app: flow-visibility
+  name: clickhouse-monitor-role-binding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: clickhouse-monitor-role
+subjects:
+  - kind: ServiceAccount
+    name: clickhouse-monitor
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  labels:
+    app: clickhouse-monitor
+  name: clickhouse-monitor
+spec:
+  schedule: "* * * * *"
+  successfulJobsHistoryLimit: 1
+  failedJobsHistoryLimit: 1
+  jobTemplate:
+    spec:
+      template:
+        metadata:
+          labels:
+            app: clickhouse-monitor
+        spec:
+          serviceAccountName: clickhouse-monitor
+          containers:
+          - name: clickhouse-monitor
+            image: flow-visibility-clickhouse-monitor
+            env:
+              - name: CH_USERNAME
+                valueFrom:
+                  secretKeyRef: 
+                    name: clickhouse-secret
+                    key: username
+              - name: CH_PASSWORD
+                valueFrom:
+                  secretKeyRef:
+                    name: clickhouse-secret
+                    key: password
+              - name: SVC_HOST
+                value: "clickhouse-clickhouse.flow-visibility.svc.cluster.local"
+              - name: SVC_PORT
+                value: "9000"
+              - name: TABLE_NAME
+                value: "default.flows"
+              - name: MV_NAMES
+                value: "default.flows_pod_view default.flows_node_view default.flows_policy_view"
+              - name: NAMESPACE
+                value: "flow-visibility"
+              - name: MONITOR_LABEL
+                value: "app=clickhouse-monitor"
+          restartPolicy: OnFailure
diff --git a/build/yamls/flow-visibility/base/provisioning/datasources/create_table.sh b/build/yamls/flow-visibility/base/provisioning/datasources/create_table.sh
@@ -68,11 +68,14 @@ clickhouse client -n -h 127.0.0.1 <<-EOSQL
         reverseThroughputFromDestinationNode UInt64,
         trusted UInt8 DEFAULT 0
     ) engine=MergeTree
-    ORDER BY (timeInserted, flowEndSeconds);
+    ORDER BY (timeInserted, flowEndSeconds)
+    TTL timeInserted + INTERVAL 1 HOUR
+    SETTINGS merge_with_ttl_timeout = 3600;
 
     CREATE MATERIALIZED VIEW flows_pod_view
     ENGINE = SummingMergeTree
     ORDER BY (
+        timeInserted,
         flowEndSeconds,
         flowEndSecondsFromSourceNode,
         flowEndSecondsFromDestinationNode,
@@ -83,8 +86,11 @@ clickhouse client -n -h 127.0.0.1 <<-EOSQL
         flowType,
         sourcePodNamespace,
         destinationPodNamespace)
+    TTL timeInserted + INTERVAL 1 HOUR
+    SETTINGS merge_with_ttl_timeout = 3600
     POPULATE
     AS SELECT
+        timeInserted,
         flowEndSeconds,
         flowEndSecondsFromSourceNode,
         flowEndSecondsFromDestinationNode,
@@ -103,6 +109,7 @@ clickhouse client -n -h 127.0.0.1 <<-EOSQL
         sum(throughputFromDestinationNode) AS throughputFromDestinationNode
     FROM flows
     GROUP BY
+        timeInserted,
         flowEndSeconds,
         flowEndSecondsFromSourceNode,
         flowEndSecondsFromDestinationNode,
@@ -117,15 +124,19 @@ clickhouse client -n -h 127.0.0.1 <<-EOSQL
     CREATE MATERIALIZED VIEW flows_node_view
     ENGINE = SummingMergeTree
     ORDER BY (
+        timeInserted,
         flowEndSeconds,
         flowEndSecondsFromSourceNode,
         flowEndSecondsFromDestinationNode,
         sourceNodeName,
         destinationNodeName,
         sourcePodNamespace,
         destinationPodNamespace)
+    TTL timeInserted + INTERVAL 1 HOUR
+    SETTINGS merge_with_ttl_timeout = 3600
     POPULATE
     AS SELECT
+        timeInserted,
         flowEndSeconds,
         flowEndSecondsFromSourceNode,
         flowEndSecondsFromDestinationNode,
@@ -143,6 +154,7 @@ clickhouse client -n -h 127.0.0.1 <<-EOSQL
         sum(reverseThroughputFromDestinationNode) AS reverseThroughputFromDestinationNode
     FROM flows
     GROUP BY
+        timeInserted,
         flowEndSeconds,
         flowEndSecondsFromSourceNode,
         flowEndSecondsFromDestinationNode,
@@ -154,6 +166,7 @@ clickhouse client -n -h 127.0.0.1 <<-EOSQL
     CREATE MATERIALIZED VIEW flows_policy_view
     ENGINE = SummingMergeTree
     ORDER BY (
+        timeInserted,
         flowEndSeconds,
         flowEndSecondsFromSourceNode,
         flowEndSecondsFromDestinationNode,
@@ -163,8 +176,11 @@ clickhouse client -n -h 127.0.0.1 <<-EOSQL
         ingressNetworkPolicyRuleAction,
         sourcePodNamespace,
         destinationPodNamespace)
+    TTL timeInserted + INTERVAL 1 HOUR
+    SETTINGS merge_with_ttl_timeout = 3600
     POPULATE
     AS SELECT
+        timeInserted,
         flowEndSeconds,
         flowEndSecondsFromSourceNode,
         flowEndSecondsFromDestinationNode,
@@ -184,6 +200,7 @@ clickhouse client -n -h 127.0.0.1 <<-EOSQL
         sum(reverseThroughputFromDestinationNode) AS reverseThroughputFromDestinationNode
     FROM flows
     GROUP BY
+        timeInserted,
         flowEndSeconds,
         flowEndSecondsFromSourceNode,
         flowEndSecondsFromDestinationNode,

diff --git a/build/yamls/flow-visibility/patches/dev/imagePullPolicy.yml b/build/yamls/flow-visibility/patches/dev/imagePullPolicy.yml
@@ -0,0 +1,42 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  labels:
+    app: clickhouse-monitor
+  name: clickhouse-monitor
+spec:
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          containers:
+          - name: clickhouse-monitor
+            imagePullPolicy: IfNotPresent
+            # kustomize has bugs in patching cronjob before version 4.3.0 which leads to 
+            # a fields-missing problem if the fields below are not present in the patch file
+            # The fields below can be deleted after kustomize is upgraded to version 4.3.0 
+            # or later in verify-kustomize.sh
+            image: flow-visibility-clickhouse-monitor
+            env:
+              - name: CH_USERNAME
+                valueFrom:
+                  secretKeyRef: 
+                    name: clickhouse-secret
+                    key: username
+              - name: CH_PASSWORD
+                valueFrom:
+                  secretKeyRef:
+                    name: clickhouse-secret
+                    key: password
+              - name: SVC_HOST
+                value: "clickhouse-clickhouse.flow-visibility.svc.cluster.local"
+              - name: SVC_PORT
+                value: "9000"
+              - name: TABLE_NAME
+                value: "default.flows"
+              - name: MV_NAMES
+                value: "default.flows_pod_view default.flows_node_view default.flows_policy_view"
+              - name: NAMESPACE
+                value: "flow-visibility"
+              - name: MONITOR_LABEL
+                value: "app=clickhouse-monitor"
diff --git a/build/yamls/flow-visibility/patches/release/.gitignore b/build/yamls/flow-visibility/patches/release/.gitignore
@@ -0,0 +1 @@
+# placeholder
diff --git a/docs/network-flow-visibility.md b/docs/network-flow-visibility.md
@@ -721,6 +721,13 @@ replicaset.apps/grafana-5c6c5b74f7   1         1         1       1m
 
 NAME                                             READY   AGE
 statefulset.apps/chi-clickhouse-clickhouse-0-0   1/1     1m
+
+
+NAME                               SCHEDULE    SUSPEND   ACTIVE   LAST SCHEDULE   AGE
+cronjob.batch/clickhouse-monitor   * * * * *   False     0        30s             1m
+
+NAME                                    COMPLETIONS   DURATION   AGE
+job.batch/clickhouse-monitor-27434986   1/1           6s         30s
 ```
 
 Run the following commands to print the IP of the workder Node and the NodePort