diff --git a/api/v1alpha1/driver_types.go b/api/v1alpha1/driver_types.go index 1084c1d3..62231764 100644 --- a/api/v1alpha1/driver_types.go +++ b/api/v1alpha1/driver_types.go @@ -33,8 +33,10 @@ const ( MonthlyPeriod PeriodicityType = "monthly" ) +// +kubebuilder:validation:XValidation:message="Either maxLogSize or periodicity must be set",rule="(has(self.maxLogSize)) || (has(self.periodicity))" type LogRotationSpec struct { // MaxFiles is the number of logrtoate files + // Default to 7 //+kubebuilder:validation:Optional MaxFiles int `json:"maxFiles,omitempty"` @@ -48,6 +50,7 @@ type LogRotationSpec struct { Periodicity PeriodicityType `json:"periodicity,omitempty"` // LogHostPath is the prefix directory path for the csi log files + // Default to /var/lib/cephcsi //+kubebuilder:validation:Optional LogHostPath string `json:"logHostPath,omitempty"` } diff --git a/config/crd/bases/csi.ceph.io_drivers.yaml b/config/crd/bases/csi.ceph.io_drivers.yaml index 24e904cd..e592d853 100644 --- a/config/crd/bases/csi.ceph.io_drivers.yaml +++ b/config/crd/bases/csi.ceph.io_drivers.yaml @@ -3542,11 +3542,14 @@ spec: description: log rotation for csi pods properties: logHostPath: - description: LogHostPath is the prefix directory path for - the csi log files + description: |- + LogHostPath is the prefix directory path for the csi log files + Default to /var/lib/cephcsi type: string maxFiles: - description: MaxFiles is the number of logrtoate files + description: |- + MaxFiles is the number of logrtoate files + Default to 7 type: integer maxLogSize: anyOf: @@ -3565,6 +3568,9 @@ spec: - monthly type: string type: object + x-kubernetes-validations: + - message: Either maxLogSize or periodicity must be set + rule: (has(self.maxLogSize)) || (has(self.periodicity)) verbosity: description: |- Log verbosity level for driver pods, diff --git a/config/crd/bases/csi.ceph.io_operatorconfigs.yaml b/config/crd/bases/csi.ceph.io_operatorconfigs.yaml index deb9e290..109d0a53 100644 --- a/config/crd/bases/csi.ceph.io_operatorconfigs.yaml +++ b/config/crd/bases/csi.ceph.io_operatorconfigs.yaml @@ -3581,11 +3581,14 @@ spec: description: log rotation for csi pods properties: logHostPath: - description: LogHostPath is the prefix directory path - for the csi log files + description: |- + LogHostPath is the prefix directory path for the csi log files + Default to /var/lib/cephcsi type: string maxFiles: - description: MaxFiles is the number of logrtoate files + description: |- + MaxFiles is the number of logrtoate files + Default to 7 type: integer maxLogSize: anyOf: @@ -3605,6 +3608,9 @@ spec: - monthly type: string type: object + x-kubernetes-validations: + - message: Either maxLogSize or periodicity must be set + rule: (has(self.maxLogSize)) || (has(self.periodicity)) verbosity: description: |- Log verbosity level for driver pods, diff --git a/docs/design/logrotate.md b/docs/design/logrotate.md index c3773fcd..b8e9e95c 100644 --- a/docs/design/logrotate.md +++ b/docs/design/logrotate.md @@ -13,14 +13,14 @@ apiVersion: csi.ceph.io/v1alpha1 spec: log: verbosity: 1 - driverSpecDefaults: + driverSpecDefaults: log: verbosity: 5 rotation: # one of: hourly, daily, weekly, monthly periodicity: daily maxLogSize: 500M - maxFiles: 5 + maxFiles: 7 logHostPath: /var/lib/cephcsi ``` @@ -35,14 +35,14 @@ metadata: spec: log: verbosity: 1 - driverSpecDefaults: + driverSpecDefaults: log: verbosity: 5 rotation: # one of: hourly, daily, weekly, monthly periodicity: daily maxLogSize: 500M - maxFiles: 5 + maxFiles: 7 logHostPath: /var/lib/cephcsi ``` @@ -51,20 +51,24 @@ Logrotator sidecar container cpu and memory usage can configured by, `OperatorConfig CRD`: ```yaml spec: - provisioner: - logRotator: - cpu: "100m" - memory: "32Mi" - plugin: - logRotator: - cpu: "100m" - memory: "32Mi" + driverSpecDefaults: + controllerPlugin: + resources: + logRotator: + cpu: "100m" + memory: "32Mi" + nodePlugin: + resources: + logRotator: + cpu: "100m" + memory: "32Mi" ``` -For systems where SELinux is enabled (e.g. OpenShift),start plugin-controller as privileged that mount a host path. +For systems where SELinux is enabled (e.g. OpenShift), start plugin-controller as privileged that mount a host path. `OperatorConfig CRD`: ```yaml spec: - provisioner: - privileged: true + driverSpecDefaults: + controllerPlugin: + privileged: true ``` diff --git a/docs/design/operator.md b/docs/design/operator.md index 01818b18..1d4f00d7 100644 --- a/docs/design/operator.md +++ b/docs/design/operator.md @@ -92,7 +92,7 @@ spec: # one of: hourly, daily, weekly, monthly periodicity: daily maxLogSize: 500M - maxFiles: 5 + maxFiles: 7 logHostPath: /var/lib/cephcsi clusterName: 5c63ad7e-74fe-4724-a511-4ccdc560da56 enableMetadata: true diff --git a/internal/controller/defaults.go b/internal/controller/defaults.go index 10ea404c..19ed50a4 100644 --- a/internal/controller/defaults.go +++ b/internal/controller/defaults.go @@ -38,8 +38,10 @@ var imageDefaults = map[string]string{ } const ( - defaultGRrpcTimeout = 150 - defaultKubeletDirPath = "/var/lib/kubelet" + defaultGRrpcTimeout = 150 + defaultKubeletDirPath = "/var/lib/kubelet" + defaultLogHostPath = "/var/lib/cephcsi" + defaultLogRotateMaxFiles = 7 ) var defaultLeaderElection = csiv1a1.LeaderElectionSpec{ diff --git a/internal/controller/driver_controller.go b/internal/controller/driver_controller.go index b766bf6f..0fca903a 100644 --- a/internal/controller/driver_controller.go +++ b/internal/controller/driver_controller.go @@ -69,8 +69,11 @@ const ( NfsDriverType = "nfs" ) -// Annotation name for ownerref information -const ownerRefAnnotationKey = "csi.ceph.io/ownerref" +const ( + // Annotation name for ownerref information + ownerRefAnnotationKey = "csi.ceph.io/ownerref" + logRotateCmd = `while true; do logrotate --verbose /logrotate-config/csi; sleep 15m; done` +) // A regexp used to parse driver's prefix and type from the full name var nameRegExp, _ = regexp.Compile(fmt.Sprintf( @@ -230,6 +233,7 @@ func (r *driverReconcile) reconcile() error { // the desired state defined on the driver object errChan := utils.RunConcurrently( r.reconcileCsiConfigMap, + r.reconcileLogRotateConfigMap, r.reconcileK8sCsiDriver, r.reconcileControllerPluginDeployment, r.reconcileNodePluginDeamonSet, @@ -327,6 +331,63 @@ func (r *driverReconcile) LoadAndValidateDesiredState() error { return nil } +func (r *driverReconcile) reconcileLogRotateConfigMap() error { + logRotateConfigmap := &corev1.ConfigMap{} + logRotateConfigmap.Name = utils.LogRotateConfigMapName(r.driver.Name) + logRotateConfigmap.Namespace = r.driver.Namespace + + log := r.log.WithValues("logRotateConfigMap", logRotateConfigmap.Name) + log.Info("Reconciling logRotate configmap") + + logRotationSpec := cmp.Or(r.driver.Spec.Log, &csiv1a1.LogSpec{}).Rotation + if logRotationSpec != nil { + opResult, err := ctrlutil.CreateOrUpdate(r.ctx, r.Client, logRotateConfigmap, func() error { + if _, err := utils.ToggleOwnerReference(true, logRotateConfigmap, &r.driver, r.Scheme); err != nil { + log.Error(err, "Failed adding an owner reference on the LogRotate config map") + return err + } + if logRotationSpec.Periodicity != "" && logRotationSpec.MaxLogSize.IsZero() { + err := fmt.Errorf("invalid Log.Rotation spec") + log.Error(err, "Either \"maxLogSize\" or \"periodicity\" fields must be set") + return err + } + maxFiles := cmp.Or(logRotationSpec.MaxFiles, defaultLogRotateMaxFiles) + cronLogRotateSettings := []string{ + "\tmissingok", + "\tcompress", + "\tcopytruncate", + "\tnotifempty", + fmt.Sprintf("\trotate %d", maxFiles), + } + if logRotationSpec.Periodicity != "" { + periodicity := "\t" + string(logRotationSpec.Periodicity) + cronLogRotateSettings = append(cronLogRotateSettings, periodicity) + } + if logRotationSpec.MaxLogSize.String() != "0" { + maxSize := fmt.Sprintf("\tmaxsize %s", logRotationSpec.MaxLogSize.String()) + cronLogRotateSettings = append(cronLogRotateSettings, maxSize) + } + logRotateConfigmap.Data = map[string]string{ + "csi": fmt.Sprintf( + "/csi-logs/*.log {\n%s\n}\n", + strings.Join(cronLogRotateSettings, "\n"), + ), + } + return nil + }) + + logCreateOrUpdateResult(log, "LogRotateConfigMap", logRotateConfigmap, opResult, err) + return err + } else { + // Remove the logrotate configmap if logrotate setting is removed from the driver's spec + if err := r.Delete(r.ctx, logRotateConfigmap); client.IgnoreNotFound(err) != nil { + log.Error(err, "Unable to delete LogRotate configmap") + return err + } + return nil + } +} + func (r *driverReconcile) reconcileCsiConfigMap() error { csiConfigMap := &corev1.ConfigMap{} csiConfigMap.Name = utils.CsiConfigVolume.Name @@ -454,6 +515,18 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error { logVerbosity := ptr.Deref(r.driver.Spec.Log, csiv1a1.LogSpec{}).Verbosity forceKernelClient := r.isCephFsDriver() && r.driver.Spec.CephFsClientType == csiv1a1.KernelCephFsClient snPolicy := cmp.Or(r.driver.Spec.SnapshotPolicy, csiv1a1.VolumeSnapshotSnapshotPolicy) + logRotationSpec := cmp.Or(r.driver.Spec.Log, &csiv1a1.LogSpec{}).Rotation + logRotationEnabled := logRotationSpec != nil + logRotateSecurityContext := utils.If( + pluginSpec.Privileged != nil && logRotationEnabled, + &corev1.SecurityContext{ + Privileged: pluginSpec.Privileged, + Capabilities: &corev1.Capabilities{ + Drop: []corev1.Capability{"All"}, + }, + }, + nil, + ) leaderElectionSettingsArg := []string{ utils.LeaderElectionNamespaceContainerArg(r.driver.Namespace), @@ -487,6 +560,7 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error { Name: fmt.Sprintf("csi-%splugin", r.driverType), Image: r.images["plugin"], ImagePullPolicy: imagePullPolicy, + SecurityContext: logRotateSecurityContext, Args: utils.DeleteZeroValues( []string{ utils.TypeContainerArg(string(r.driverType)), @@ -504,6 +578,13 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error { utils.CsiAddonsEndpointContainerArg, "", ), + utils.If(logRotationEnabled, utils.LogToStdErrContainerArg, ""), + utils.If(logRotationEnabled, utils.AlsoLogToStdErrContainerArg, ""), + utils.If( + logRotationEnabled, + utils.LogFileContainerArg(fmt.Sprintf("csi-%splugin", r.driverType)), + "", + ), }, ), Env: []corev1.EnvVar{ @@ -534,6 +615,9 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error { if r.isRdbDriver() { mounts = append(mounts, utils.OidcTokenVolumeMount) } + if logRotationEnabled { + mounts = append(mounts, utils.LogsDirVolumeMount) + } return mounts }), Resources: ptr.Deref( @@ -654,6 +738,7 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error { Name: "csi-addons", Image: r.images["addons"], ImagePullPolicy: imagePullPolicy, + SecurityContext: logRotateSecurityContext, Args: utils.DeleteZeroValues( append( slices.Clone(leaderElectionSettingsArg), @@ -664,6 +749,9 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error { utils.CsiAddonsAddressContainerArg, utils.ControllerPortContainerArg, utils.NamespaceContainerArg, + utils.If(logRotationEnabled, utils.LogToStdErrContainerArg, ""), + utils.If(logRotationEnabled, utils.AlsoLogToStdErrContainerArg, ""), + utils.If(logRotationEnabled, utils.LogFileContainerArg("csi-addons"), ""), ), ), Ports: []corev1.ContainerPort{ @@ -675,9 +763,15 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error { utils.PodNameEnvVar, utils.PodNamespaceEnvVar, }, - VolumeMounts: []corev1.VolumeMount{ - utils.SocketDirVolumeMount, - }, + VolumeMounts: utils.Call(func() []corev1.VolumeMount { + mounts := []corev1.VolumeMount{ + utils.SocketDirVolumeMount, + } + if logRotationEnabled { + mounts = append(mounts, utils.LogsDirVolumeMount) + } + return mounts + }), Resources: ptr.Deref( pluginSpec.Resources.Addons, corev1.ResourceRequirements{}, @@ -741,7 +835,22 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error { ), }) } - + // CSI LogRotate Container + if logRotationEnabled { + resources := ptr.Deref(pluginSpec.Resources.LogRotator, corev1.ResourceRequirements{}) + containers = append(containers, corev1.Container{ + Name: "log-rotator", + Image: r.images["plugin"], + ImagePullPolicy: imagePullPolicy, + Resources: resources, + SecurityContext: logRotateSecurityContext, + Command: []string{"/bin/bash", "-c", logRotateCmd}, + VolumeMounts: []corev1.VolumeMount{ + utils.LogsDirVolumeMount, + utils.LogRotateDirVolumeMount, + }, + }) + } return containers }), Volumes: utils.Call(func() []corev1.Volume { @@ -767,6 +876,14 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error { volumes, utils.KmsConfigVolume(&r.driver.Spec.Encryption.ConfigMapRef)) } + if logRotationEnabled { + logHostPath := cmp.Or(logRotationSpec.LogHostPath, defaultLogHostPath) + volumes = append( + volumes, + utils.LogsDirVolume(logHostPath, deploy.Name), + utils.LogRotateDirVolumeName(r.driver.Name), + ) + } return volumes }), }, @@ -807,6 +924,8 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error { topology := r.isRdbDriver() && pluginSpec.Topology != nil domainLabels := cmp.Or(pluginSpec.Topology, &csiv1a1.TopologySpec{}).DomainLabels + logRotationSpec := cmp.Or(r.driver.Spec.Log, &csiv1a1.LogSpec{}).Rotation + logRotationEnabled := logRotationSpec != nil daemonSet.Spec = appsv1.DaemonSetSpec{ Selector: &metav1.LabelSelector{ @@ -884,6 +1003,13 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error { utils.DomainLabelsContainerArg(domainLabels), "", ), + utils.If(logRotationEnabled, utils.LogToStdErrContainerArg, ""), + utils.If(logRotationEnabled, utils.AlsoLogToStdErrContainerArg, ""), + utils.If( + logRotationEnabled, + utils.LogFileContainerArg(fmt.Sprintf("csi-%splugin", r.driverType)), + "", + ), }, ), Env: []corev1.EnvVar{ @@ -912,6 +1038,9 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error { if r.isRdbDriver() { mounts = append(mounts, utils.OidcTokenVolumeMount) } + if logRotationEnabled { + mounts = append(mounts, utils.LogsDirVolumeMount) + } return mounts }), Resources: ptr.Deref( @@ -965,6 +1094,7 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error { Args: utils.DeleteZeroValues( []string{ utils.CsiAddonsNodeIdContainerArg, + utils.NodeIdContainerArg, utils.LogVerbosityContainerArg(logVerbosity), utils.CsiAddonsAddressContainerArg, utils.ControllerPortContainerArg, @@ -972,6 +1102,9 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error { utils.NamespaceContainerArg, utils.PodUidContainerArg, utils.StagingPathContainerArg(kubeletDirPath), + utils.If(logRotationEnabled, utils.LogToStdErrContainerArg, ""), + utils.If(logRotationEnabled, utils.AlsoLogToStdErrContainerArg, ""), + utils.If(logRotationEnabled, utils.LogFileContainerArg("csi-addons"), ""), }, ), Ports: []corev1.ContainerPort{ @@ -983,9 +1116,15 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error { utils.PodNamespaceEnvVar, utils.PodUidEnvVar, }, - VolumeMounts: []corev1.VolumeMount{ - utils.PluginDirVolumeMount, - }, + VolumeMounts: utils.Call(func() []corev1.VolumeMount { + mounts := []corev1.VolumeMount{ + utils.PluginDirVolumeMount, + } + if logRotationEnabled { + mounts = append(mounts, utils.LogsDirVolumeMount) + } + return mounts + }), Resources: ptr.Deref( pluginSpec.Resources.Addons, corev1.ResourceRequirements{}, @@ -1026,6 +1165,27 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error { }) } } + // CSI LogRotate Container + if logRotationEnabled { + resources := ptr.Deref(pluginSpec.Resources.LogRotator, corev1.ResourceRequirements{}) + containers = append(containers, corev1.Container{ + Name: "log-rotator", + Image: r.images["plugin"], + ImagePullPolicy: imagePullPolicy, + Resources: resources, + SecurityContext: &corev1.SecurityContext{ + Privileged: ptr.To(true), + Capabilities: &corev1.Capabilities{ + Drop: []corev1.Capability{"All"}, + }, + }, + Command: []string{"/bin/bash", "-c", logRotateCmd}, + VolumeMounts: []corev1.VolumeMount{ + utils.LogsDirVolumeMount, + utils.LogRotateDirVolumeMount, + }, + }) + } return containers }), Volumes: utils.Call(func() []corev1.Volume { @@ -1067,6 +1227,14 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error { utils.OidcTokenVolume, ) } + if logRotationEnabled { + logHostPath := cmp.Or(logRotationSpec.LogHostPath, defaultLogHostPath) + volumes = append( + volumes, + utils.LogsDirVolume(logHostPath, daemonSet.Name), + utils.LogRotateDirVolumeName(r.driver.Name), + ) + } return volumes }), }, @@ -1273,6 +1441,9 @@ func mergeDriverSpecs(dest, src *csiv1a1.DriverSpec) { if dest.Resources.Plugin == nil { dest.Resources.Plugin = src.Resources.Plugin } + if dest.Resources.LogRotator == nil { + dest.Resources.LogRotator = src.Resources.LogRotator + } } } if src.ControllerPlugin != nil { @@ -1307,6 +1478,9 @@ func mergeDriverSpecs(dest, src *csiv1a1.DriverSpec) { if dest.Replicas == nil { dest.Replicas = src.Replicas } + if dest.Privileged == nil { + dest.Privileged = src.Privileged + } if dest.Resources.Attacher == nil { dest.Resources.Attacher = src.Resources.Attacher } @@ -1328,6 +1502,9 @@ func mergeDriverSpecs(dest, src *csiv1a1.DriverSpec) { if dest.Resources.Plugin == nil { dest.Resources.Plugin = src.Resources.Plugin } + if dest.Resources.LogRotator == nil { + dest.Resources.LogRotator = src.Resources.LogRotator + } } } if dest.AttachRequired == nil { diff --git a/internal/utils/csi.go b/internal/utils/csi.go index 0df4ae77..4ab0521f 100644 --- a/internal/utils/csi.go +++ b/internal/utils/csi.go @@ -37,6 +37,9 @@ const ( CsiConfigMapConfigKey = "config.json" CsiConfigMapMappingKey = "cluster-mapping.json" + + logsDirVolumeName = "logs-dir" + logRotateDirVolumeName = "log-rotate-dir" ) // Ceph CSI common volumes @@ -182,6 +185,29 @@ func PodsMountDirVolume(kubeletDirPath string) corev1.Volume { }, } } +func LogsDirVolume(logHostPath, pluginName string) corev1.Volume { + return corev1.Volume{ + Name: logsDirVolumeName, + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: fmt.Sprintf("%s/%s", logHostPath, pluginName), + Type: ptr.To(corev1.HostPathDirectoryOrCreate), + }, + }, + } +} +func LogRotateDirVolumeName(driverName string) corev1.Volume { + return corev1.Volume{ + Name: logRotateDirVolumeName, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: LogRotateConfigMapName(driverName), + }, + }, + }, + } +} // Ceph CSI common volume Mounts var SocketDirVolumeMount = corev1.VolumeMount{ @@ -236,6 +262,14 @@ var EtcSelinuxVolumeMount = corev1.VolumeMount{ MountPath: "/etc/selinux", ReadOnly: true, } +var LogsDirVolumeMount = corev1.VolumeMount{ + Name: logsDirVolumeName, + MountPath: "/csi-logs", +} +var LogRotateDirVolumeMount = corev1.VolumeMount{ + Name: logRotateDirVolumeName, + MountPath: "/logrotate-config", +} func PodsMountDirVolumeMount(kubletDirPath string) corev1.VolumeMount { return corev1.VolumeMount{ @@ -335,10 +369,15 @@ var ImmediateTopologyContainerArg = "--immediate-topology=false" var RecoverVolumeExpansionFailureContainerArg = "--feature-gates=RecoverVolumeExpansionFailure=true" var EnableVolumeGroupSnapshotsContainerArg = "--enable-volume-group-snapshots=true" var ForceCephKernelClientContainerArg = "--forcecephkernelclient=true" +var LogToStdErrContainerArg = "--logtostderr=false" +var AlsoLogToStdErrContainerArg = "--alsologtostderr=true" func LogVerbosityContainerArg(level int) string { return fmt.Sprintf("--v=%d", Clamp(level, 0, 5)) } +func LogFileContainerArg(containerName string) string { + return fmt.Sprintf("--log_file=csi-logs/%s.log", containerName) +} func TypeContainerArg(t string) string { switch t { case "rbd", "cephfs", "nfs", "controller", "liveness": @@ -380,6 +419,9 @@ func KubeletRegistrationPathContainerArg(kubeletDirPath string, driverName strin func StagingPathContainerArg(kubeletDirPath string) string { return fmt.Sprintf("--stagingpath=%s/plugins/kubernetes.io/csi/", kubeletDirPath) } +func LogRotateConfigMapName(driverName string) string { + return fmt.Sprintf("%s-logrotate-config", driverName) +} func KernelMountOptionsContainerArg(options map[string]string) string { return If( len(options) > 0,