Support log file size and number limits (#879)

klog does not enforce max log file size limit when the log file is not specifiec by the --log_file argument. This commit sets klog.MaxSize to the --log_file_max_size argument value when logging to file is enabled and --log_file is not provided. The commit also adds a new argument --log_file_max_num to define the max number (per severity level) of log files to be kept. The file number limit is enforced by Antrea code that periodically checks the log files and deletes the oldest files to keep at most the specified max number of log files. Fixes: #788
antrea-io · Jul 2, 2020 · cfd6440 · cfd6440
1 parent 0448530
commit cfd6440
Show file tree

Hide file tree

Showing 14 changed files with 430 additions and 24 deletions.
diff --git a/build/yamls/antrea-eks.yml b/build/yamls/antrea-eks.yml
@@ -662,9 +662,10 @@ spec:
         - --config
         - /etc/antrea/antrea-controller.conf
         - --logtostderr=false
-        - --log_dir
-        - /var/log/antrea
+        - --log_dir=/var/log/antrea
         - --alsologtostderr
+        - --log_file_max_size=100
+        - --log_file_max_num=4
         command:
         - antrea-controller
         env:
@@ -787,9 +788,10 @@ spec:
         - --config
         - /etc/antrea/antrea-agent.conf
         - --logtostderr=false
-        - --log_dir
-        - /var/log/antrea
+        - --log_dir=/var/log/antrea
         - --alsologtostderr
+        - --log_file_max_size=100
+        - --log_file_max_num=4
         command:
         - antrea-agent
         env:

diff --git a/build/yamls/antrea-gke.yml b/build/yamls/antrea-gke.yml
@@ -662,9 +662,10 @@ spec:
         - --config
         - /etc/antrea/antrea-controller.conf
         - --logtostderr=false
-        - --log_dir
-        - /var/log/antrea
+        - --log_dir=/var/log/antrea
         - --alsologtostderr
+        - --log_file_max_size=100
+        - --log_file_max_num=4
         command:
         - antrea-controller
         env:
@@ -787,9 +788,10 @@ spec:
         - --config
         - /etc/antrea/antrea-agent.conf
         - --logtostderr=false
-        - --log_dir
-        - /var/log/antrea
+        - --log_dir=/var/log/antrea
         - --alsologtostderr
+        - --log_file_max_size=100
+        - --log_file_max_num=4
         command:
         - antrea-agent
         env:

diff --git a/build/yamls/antrea-ipsec.yml b/build/yamls/antrea-ipsec.yml
@@ -671,9 +671,10 @@ spec:
         - --config
         - /etc/antrea/antrea-controller.conf
         - --logtostderr=false
-        - --log_dir
-        - /var/log/antrea
+        - --log_dir=/var/log/antrea
         - --alsologtostderr
+        - --log_file_max_size=100
+        - --log_file_max_num=4
         command:
         - antrea-controller
         env:
@@ -826,9 +827,10 @@ spec:
         - --config
         - /etc/antrea/antrea-agent.conf
         - --logtostderr=false
-        - --log_dir
-        - /var/log/antrea
+        - --log_dir=/var/log/antrea
         - --alsologtostderr
+        - --log_file_max_size=100
+        - --log_file_max_num=4
         command:
         - antrea-agent
         env:

diff --git a/build/yamls/antrea-windows.yml b/build/yamls/antrea-windows.yml
@@ -1,16 +1,16 @@
 apiVersion: v1
 data:
-  Run-AntreaAgent.ps1: |-
+  Run-AntreaAgent.ps1: |
     $ErrorActionPreference = "Stop"
     # wins will rename the binary when executing it. So we need to copy the binary everytime before running it.
     mkdir -force /host/k/antrea/bin
     cp /k/antrea/bin/* /host/k/antrea/bin/
-    C:/k/antrea/utils/wins.exe cli process run --path /k/antrea/bin/antrea-agent.exe --args "--config=/k/antrea/etc/antrea-agent.conf --logtostderr=false --log_dir=/k/antrea/logs/ --alsologtostderr" --envs "KUBERNETES_SERVICE_HOST=$env:KUBERNETES_SERVICE_HOST KUBERNETES_SERVICE_PORT=$env:KUBERNETES_SERVICE_PORT ANTREA_SERVICE_HOST=$env:ANTREA_SERVICE_HOST ANTREA_SERVICE_PORT=$env:ANTREA_SERVICE_PORT NODE_NAME=$env:NODE_NAME"
+    C:/k/antrea/utils/wins.exe cli process run --path /k/antrea/bin/antrea-agent.exe --args "--config=/k/antrea/etc/antrea-agent.conf --logtostderr=false --log_dir=/k/antrea/logs/ --alsologtostderr --log_file_max_size=100 --log_file_max_num=4" --envs "KUBERNETES_SERVICE_HOST=$env:KUBERNETES_SERVICE_HOST KUBERNETES_SERVICE_PORT=$env:KUBERNETES_SERVICE_PORT ANTREA_SERVICE_HOST=$env:ANTREA_SERVICE_HOST ANTREA_SERVICE_PORT=$env:ANTREA_SERVICE_PORT NODE_NAME=$env:NODE_NAME"
 kind: ConfigMap
 metadata:
   labels:
     app: antrea
-  name: antrea-agent-windows-9ffbt88hmc
+  name: antrea-agent-windows-59cc7mmd6h
   namespace: kube-system
 ---
 apiVersion: v1
@@ -161,7 +161,7 @@ spec:
         name: antrea-windows-config
       - configMap:
           defaultMode: 420
-          name: antrea-agent-windows-9ffbt88hmc
+          name: antrea-agent-windows-59cc7mmd6h
         name: antrea-agent-windows
       - hostPath:
           path: /etc/cni/net.d

diff --git a/build/yamls/antrea.yml b/build/yamls/antrea.yml
@@ -662,9 +662,10 @@ spec:
         - --config
         - /etc/antrea/antrea-controller.conf
         - --logtostderr=false
-        - --log_dir
-        - /var/log/antrea
+        - --log_dir=/var/log/antrea
         - --alsologtostderr
+        - --log_file_max_size=100
+        - --log_file_max_num=4
         command:
         - antrea-controller
         env:
@@ -787,9 +788,10 @@ spec:
         - --config
         - /etc/antrea/antrea-agent.conf
         - --logtostderr=false
-        - --log_dir
-        - /var/log/antrea
+        - --log_dir=/var/log/antrea
         - --alsologtostderr
+        - --log_file_max_size=100
+        - --log_file_max_num=4
         command:
         - antrea-agent
         env:

diff --git a/build/yamls/base/agent.yml b/build/yamls/base/agent.yml
@@ -69,7 +69,7 @@ spec:
               cpu: "200m"
           command: ["antrea-agent"]
           # Log to both "/var/log/antrea/" and stderr (so "kubectl logs" can work).
-          args: ["--config", "/etc/antrea/antrea-agent.conf", "--logtostderr=false", "--log_dir", "/var/log/antrea", "--alsologtostderr"]
+          args: ["--config", "/etc/antrea/antrea-agent.conf", "--logtostderr=false", "--log_dir=/var/log/antrea", "--alsologtostderr", "--log_file_max_size=100", "--log_file_max_num=4"]
           env:
             # Provide pod and node information for clusterinformation CRD.
             - name: POD_NAME

diff --git a/build/yamls/base/controller.yml b/build/yamls/base/controller.yml
@@ -80,7 +80,7 @@ spec:
               cpu: "200m"
           command: ["antrea-controller"]
           # Log to both "/var/log/antrea/" and stderr (so "kubectl logs" can work).
-          args: ["--config", "/etc/antrea/antrea-controller.conf", "--logtostderr=false", "--log_dir", "/var/log/antrea", "--alsologtostderr"]
+          args: ["--config", "/etc/antrea/antrea-controller.conf", "--logtostderr=false", "--log_dir=/var/log/antrea", "--alsologtostderr", "--log_file_max_size=100", "--log_file_max_num=4"]
           env:
             # Provide pod and node information for clusterinformation CRD.
             - name: POD_NAME

diff --git a/build/yamls/windows/base/conf/Run-AntreaAgent.ps1 b/build/yamls/windows/base/conf/Run-AntreaAgent.ps1
@@ -2,4 +2,4 @@ $ErrorActionPreference = "Stop"
 # wins will rename the binary when executing it. So we need to copy the binary everytime before running it.
 mkdir -force /host/k/antrea/bin
 cp /k/antrea/bin/* /host/k/antrea/bin/
-C:/k/antrea/utils/wins.exe cli process run --path /k/antrea/bin/antrea-agent.exe --args "--config=/k/antrea/etc/antrea-agent.conf --logtostderr=false --log_dir=/k/antrea/logs/ --alsologtostderr" --envs "KUBERNETES_SERVICE_HOST=$env:KUBERNETES_SERVICE_HOST KUBERNETES_SERVICE_PORT=$env:KUBERNETES_SERVICE_PORT ANTREA_SERVICE_HOST=$env:ANTREA_SERVICE_HOST ANTREA_SERVICE_PORT=$env:ANTREA_SERVICE_PORT NODE_NAME=$env:NODE_NAME"
+C:/k/antrea/utils/wins.exe cli process run --path /k/antrea/bin/antrea-agent.exe --args "--config=/k/antrea/etc/antrea-agent.conf --logtostderr=false --log_dir=/k/antrea/logs/ --alsologtostderr --log_file_max_size=100 --log_file_max_num=4" --envs "KUBERNETES_SERVICE_HOST=$env:KUBERNETES_SERVICE_HOST KUBERNETES_SERVICE_PORT=$env:KUBERNETES_SERVICE_PORT ANTREA_SERVICE_HOST=$env:ANTREA_SERVICE_HOST ANTREA_SERVICE_PORT=$env:ANTREA_SERVICE_PORT NODE_NAME=$env:NODE_NAME"
diff --git a/cmd/antrea-agent/agent.go b/cmd/antrea-agent/agent.go
@@ -40,6 +40,7 @@ import (
 	crdinformers "github.com/vmware-tanzu/antrea/pkg/client/informers/externalversions"
 	"github.com/vmware-tanzu/antrea/pkg/features"
 	"github.com/vmware-tanzu/antrea/pkg/k8s"
+	"github.com/vmware-tanzu/antrea/pkg/log"
 	"github.com/vmware-tanzu/antrea/pkg/monitor"
 	ofconfig "github.com/vmware-tanzu/antrea/pkg/ovs/openflow"
 	"github.com/vmware-tanzu/antrea/pkg/ovs/ovsconfig"
@@ -186,6 +187,8 @@ func run(o *Options) error {
 	// exits, we will force exit.
 	stopCh := signals.RegisterSignalHandlers()
 
+	log.StartLogFileNumberMonitor(stopCh)
+
 	go cniServer.Run(stopCh)
 
 	informerFactory.Start(stopCh)

diff --git a/cmd/antrea-agent/main.go b/cmd/antrea-agent/main.go
@@ -25,6 +25,7 @@ import (
 	"k8s.io/component-base/logs"
 	"k8s.io/klog"
 
+	"github.com/vmware-tanzu/antrea/pkg/log"
 	"github.com/vmware-tanzu/antrea/pkg/version"
 )
 
@@ -33,7 +34,6 @@ func main() {
 	defer logs.FlushLogs()
 
 	command := newAgentCommand()
-
 	if err := command.Execute(); err != nil {
 		logs.FlushLogs()
 		os.Exit(1)
@@ -47,6 +47,7 @@ func newAgentCommand() *cobra.Command {
 		Use:  "antrea-agent",
 		Long: "The Antrea agent runs on each node.",
 		Run: func(cmd *cobra.Command, args []string) {
+			log.InitLogFileLimits(cmd.Flags())
 			if err := opts.complete(args); err != nil {
 				klog.Fatalf("Failed to complete: %v", err)
 			}
@@ -62,6 +63,7 @@ func newAgentCommand() *cobra.Command {
 
 	flags := cmd.Flags()
 	opts.addFlags(flags)
+	log.AddFlags(flags)
 	// Install log flags
 	flags.AddGoFlagSet(flag.CommandLine)
 	return cmd

diff --git a/cmd/antrea-controller/controller.go b/cmd/antrea-controller/controller.go
@@ -42,6 +42,7 @@ import (
 	"github.com/vmware-tanzu/antrea/pkg/controller/traceflow"
 	"github.com/vmware-tanzu/antrea/pkg/features"
 	"github.com/vmware-tanzu/antrea/pkg/k8s"
+	"github.com/vmware-tanzu/antrea/pkg/log"
 	"github.com/vmware-tanzu/antrea/pkg/monitor"
 	"github.com/vmware-tanzu/antrea/pkg/signals"
 	"github.com/vmware-tanzu/antrea/pkg/version"
@@ -118,6 +119,8 @@ func run(o *Options) error {
 	// exits, we will force exit.
 	stopCh := signals.RegisterSignalHandlers()
 
+	log.StartLogFileNumberMonitor(stopCh)
+
 	informerFactory.Start(stopCh)
 	crdInformerFactory.Start(stopCh)
 

diff --git a/cmd/antrea-controller/main.go b/cmd/antrea-controller/main.go
@@ -25,6 +25,7 @@ import (
 	"k8s.io/component-base/logs"
 	"k8s.io/klog"
 
+	"github.com/vmware-tanzu/antrea/pkg/log"
 	"github.com/vmware-tanzu/antrea/pkg/version"
 )
 
@@ -47,6 +48,7 @@ func newControllerCommand() *cobra.Command {
 		Use:  "antrea-controller",
 		Long: "The Antrea Controller.",
 		Run: func(cmd *cobra.Command, args []string) {
+			log.InitLogFileLimits(cmd.Flags())
 			if err := opts.complete(args); err != nil {
 				klog.Fatalf("Failed to complete: %v", err)
 			}
@@ -62,6 +64,7 @@ func newControllerCommand() *cobra.Command {
 
 	flags := cmd.Flags()
 	opts.addFlags(flags)
+	log.AddFlags(flags)
 	// Install log flags
 	flags.AddGoFlagSet(flag.CommandLine)
 	return cmd