Skip to content

Commit

Permalink
logrotate: add logrotate functionality for csi
Browse files Browse the repository at this point in the history 
1) Make main container and csi addons container
   log to a file(dependency on klog)

2) Add a log-rotate sidecar container,
   so it can rotate the logs

3) Added other volume and volumemounts as needed

4) Added the privileged option for controllerplugin

5) Add resources to the logrotate container

6) make the api change in the merge function

Signed-off-by: parth-gr <partharora1010@gmail.com>
  • Loading branch information
@parth-gr
parth-gr committed Jul 23, 2024
1 parent badf2d6 commit 10e384a
Show file tree
Hide file tree
Showing 2 changed files with 144 additions and 6 deletions.
94 changes: 93 additions & 1 deletion internal/controller/driver_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import (
"reflect"
"regexp"
"slices"
"strconv"
"strings"

"github.com/go-logr/logr"
Expand Down Expand Up @@ -461,6 +462,7 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error {
Name: fmt.Sprintf("csi-%splugin", r.driverType),
Image: r.images["plugin"],
ImagePullPolicy: imagePullPolicy,
SecurityContext: utils.If(r.logRotation(), utils.Privileged(), &corev1.SecurityContext{Privileged: ptr.To(false)}),
Args: []string{
utils.TypeContainerArg(string(r.driverType)),
utils.LogLevelContainerArg(logLevel),
Expand All @@ -477,6 +479,9 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error {
utils.CsiAddonsEndpointContainerArg,
"",
),
utils.If(r.logRotation(), utils.LogRotationSpecArg(r.driver.Spec.Log.Rotation.LogHostPath, deploy.Name, fmt.Sprintf("csi-%splugin", r.driverType)), ""),
utils.If(r.logRotation(), utils.LogToFile, ""),
utils.If(r.logRotation(), utils.LogToStderr, ""),
},
Env: []corev1.EnvVar{
utils.PodIpEnvVar,
Expand All @@ -499,6 +504,7 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error {
utils.LibModulesVolumeMount,
utils.KeysTmpDirVolumeMount,
utils.CsiConfigVolumeMount,
utils.If(r.logRotation(), utils.LogDirVolumeMount(r.driver.Spec.Log.Rotation.LogHostPath, deploy.Name), corev1.VolumeMount{}),
)
if r.driver.Spec.Encryption != nil {
mounts = append(mounts, utils.KmsConfigVolumeMount)
Expand Down Expand Up @@ -612,6 +618,7 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error {
Name: "csi-addons",
Image: r.images["addons"],
ImagePullPolicy: imagePullPolicy,
SecurityContext: utils.If(r.logRotation(), utils.Privileged(), &corev1.SecurityContext{Privileged: ptr.To(false)}),
Args: append(
slices.Clone(leaderElectionArgs),
utils.LogLevelContainerArg(logLevel),
Expand All @@ -621,6 +628,9 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error {
utils.CsiAddonsAddressContainerArg,
utils.ControllerPortContainerArg,
utils.NamespaceContainerArg,
utils.If(r.logRotation(), utils.LogRotationSpecArg(r.driver.Spec.Log.Rotation.LogHostPath, deploy.Name, "csi-addons"), ""),
utils.If(r.logRotation(), utils.LogToFile, ""),
utils.If(r.logRotation(), utils.LogToStderr, ""),
),
Ports: []corev1.ContainerPort{
utils.CsiAddonsContainerPort,
Expand All @@ -633,6 +643,7 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error {
},
VolumeMounts: []corev1.VolumeMount{
utils.SocketDirVolumeMount,
utils.If(r.logRotation(), utils.LogDirVolumeMount(r.driver.Spec.Log.Rotation.LogHostPath, deploy.Name), corev1.VolumeMount{}),
},
Resources: ptr.Deref(
pluginSpec.Resources.Addons,
Expand Down Expand Up @@ -693,7 +704,10 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error {
),
})
}

// CSI Logrotate Container
if r.driver.Spec.Log.Rotation != nil {
containers = append(containers, r.logrRotateContainer(imagePullPolicy, deploy.Name, r.driver.Spec.ControllerPlugin.Resources.LogRotator))
}
return containers
}),
Volumes: utils.Call(func() []corev1.Volume {
Expand All @@ -713,6 +727,8 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error {
utils.KeysTmpDirVolume,
utils.OidcTokenVolume,
utils.CsiConfigVolume,
utils.If(r.logRotation(), utils.LogDirVolume(r.driver.Spec.Log.Rotation.LogHostPath, deploy.Name), corev1.Volume{}),
utils.If(r.logRotation(), utils.LogSidecarDirVolume(r.driver.Spec.Log.Rotation.LogHostPath, deploy.Name), corev1.Volume{}),
)
if r.driver.Spec.Encryption != nil {
volumes = append(
Expand Down Expand Up @@ -811,6 +827,9 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error {
utils.If(r.isRdbDriver(), utils.StagingPathContainerArg(kubeletDirPath), ""),
utils.If(r.isCephFsDriver(), utils.KernelMountOptionsContainerArg(r.driver.Spec.KernelMountOptions), ""),
utils.If(r.isCephFsDriver(), utils.FuseMountOptionsContainerArg(r.driver.Spec.FuseMountOptions), ""),
utils.If(r.logRotation(), utils.LogRotationSpecArg(r.driver.Spec.Log.Rotation.LogHostPath, daemonSet.Name, fmt.Sprintf("csi-%splugin", r.driverType)), ""),
utils.If(r.logRotation(), utils.LogToFile, ""),
utils.If(r.logRotation(), utils.LogToStderr, ""),
// TODO: RBD only, add "--domainlabels={{ .CSIDomainLabels }}". not sure hot to get the info
},
Env: []corev1.EnvVar{
Expand All @@ -828,6 +847,7 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error {
utils.PluginDirVolumeMount,
utils.PluginMountDirVolumeMount(kubeletDirPath),
utils.PodsMountDirVolumeMount(kubeletDirPath),
utils.If(r.logRotation(), utils.LogDirVolumeMount(r.driver.Spec.Log.Rotation.LogHostPath, daemonSet.Name), corev1.VolumeMount{}),
}
if ptr.Deref(pluginSpec.EnableSeLinuxHostMount, false) {
mounts = append(mounts, utils.EtcSelinuxVolumeMount)
Expand Down Expand Up @@ -895,6 +915,9 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error {
utils.NamespaceContainerArg,
utils.PodUidContainerArg,
utils.StagingPathContainerArg(kubeletDirPath),
utils.If(r.logRotation(), utils.LogRotationSpecArg(r.driver.Spec.Log.Rotation.LogHostPath, daemonSet.Name, "csi-addons"), ""),
utils.If(r.logRotation(), utils.LogToFile, ""),
utils.If(r.logRotation(), utils.LogToStderr, ""),
},
Ports: []corev1.ContainerPort{
utils.CsiAddonsContainerPort,
Expand All @@ -907,6 +930,7 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error {
},
VolumeMounts: []corev1.VolumeMount{
utils.PluginDirVolumeMount,
utils.If(r.logRotation(), utils.LogDirVolumeMount(r.driver.Spec.Log.Rotation.LogHostPath, daemonSet.Name), corev1.VolumeMount{}),
},
Resources: ptr.Deref(
pluginSpec.Resources.Addons,
Expand Down Expand Up @@ -946,6 +970,10 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error {
})
}
}
// CSI Logrotate Container
if r.driver.Spec.Log.Rotation != nil {
containers = append(containers, r.logrRotateContainer(imagePullPolicy, daemonSet.Name, r.driver.Spec.NodePlugin.Resources.LogRotator))
}
return containers
}),
Volumes: utils.Call(func() []corev1.Volume {
Expand All @@ -967,6 +995,8 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error {
utils.PluginMountDirVolume(pluginSpec.KubeletDirPath),
utils.PodsMountDirVolume(pluginSpec.KubeletDirPath),
utils.RegistrationDirVolume(pluginSpec.KubeletDirPath),
utils.If(r.logRotation(), utils.LogDirVolume(r.driver.Spec.Log.Rotation.LogHostPath, daemonSet.Name), corev1.Volume{}),
utils.If(r.logRotation(), utils.LogSidecarDirVolume(r.driver.Spec.Log.Rotation.LogHostPath, daemonSet.Name), corev1.Volume{}),
)
if ptr.Deref(pluginSpec.EnableSeLinuxHostMount, false) {
volumes = append(
Expand Down Expand Up @@ -1053,6 +1083,10 @@ func (r *driverReconcile) isNfsDriver() bool {
return r.driverType == NfsDriverType
}

func (r *driverReconcile) logRotation() bool {
return r.driver.Spec.Log.Rotation != nil
}

func (r *driverReconcile) generateName(suffix string) string {
return fmt.Sprintf("%s-%s", r.driver.Name, suffix)
}
Expand Down Expand Up @@ -1183,6 +1217,9 @@ func mergeDriverSpecs(dest, src *csiv1a1.DriverSpec) {
if dest.Resources.Plugin == nil {
dest.Resources.Plugin = src.Resources.Plugin
}
if dest.Resources.LogRotator == nil {
dest.Resources.LogRotator = src.Resources.LogRotator
}
}
}
if src.ControllerPlugin != nil {
Expand Down Expand Up @@ -1238,6 +1275,9 @@ func mergeDriverSpecs(dest, src *csiv1a1.DriverSpec) {
if dest.Resources.Plugin == nil {
dest.Resources.Plugin = src.Resources.Plugin
}
if dest.Resources.LogRotator == nil {
dest.Resources.LogRotator = src.Resources.LogRotator
}
}
}
if dest.AttachRequired == nil {
Expand All @@ -1259,3 +1299,55 @@ func mergeDriverSpecs(dest, src *csiv1a1.DriverSpec) {
dest.CephFsClientType = src.CephFsClientType
}
}

func (r *driverReconcile) logrRotateContainer(imagePullPolicy corev1.PullPolicy, csiComponentName string, resources *corev1.ResourceRequirements) corev1.Container {
rotation := r.driver.Spec.Log.Rotation
return corev1.Container{
Name: "log-collector",
Image: r.images["plugin"],
ImagePullPolicy: imagePullPolicy,
Resources: ptr.Deref(
resources,
corev1.ResourceRequirements{},
),
Command: []string{
"/bin/bash",
"-c", // Command to run
fmt.Sprintf(cronLogRotate, rotation.Periodicity, rotation.MaxLogSize.String(), strconv.Itoa(rotation.MaxFiles), rotation.LogHostPath, csiComponentName),
},
VolumeMounts: []corev1.VolumeMount{
utils.LogDirVolumeMount(r.driver.Spec.Log.Rotation.LogHostPath, csiComponentName),
utils.LogSidecarVolumeMount(r.driver.Spec.Log.Rotation.LogHostPath, csiComponentName),
},
}
}

var cronLogRotate = `
echo "Starting the csi-logrotate-sidecar"
PERIODICITY=%s
LOG_MAX_SIZE=%s
ROTATE=%s
CsiLogHostPath=%s
CsiComponentName=%s
mkdir -p "$CsiLogHostPath"/logrotate-config/"$CsiComponentName"
echo '"$CsiLogHostPath"/log/"$CsiComponentName"/*.log {
"$PERIODICITY"
missingok
rotate "$ROTATE"
compress
copytruncate
notifempty
}' > "$CsiLogHostPath"/logrotate-config/"$CsiComponentName"/csi
echo "File creation container completed"
LOG_ROTATE_CEPH_CSI_FILE="$CsiLogHostPath"/logrotate-config/"$CsiComponentName"/csi
if [ "$LOG_MAX_SIZE" != "0" ]; then
sed --in-place "4i \ \ \ \ maxsize $LOG_MAX_SIZE" "$LOG_ROTATE_CEPH_CSI_FILE"
fi
while true; do
logrotate --verbose "$LOG_ROTATE_CEPH_CSI_FILE"
sleep 15m
done
`
56 changes: 51 additions & 5 deletions internal/utils/csi.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,11 +28,13 @@ const (
csiEndpoint = "unix://" + SocketDir + "/csi.sock"
csiAddonsEndpoint = "unix://" + SocketDir + "/csi-addons.sock.sock"

kmsConfigVolumeName = "ceph-csi-kms-config"
registrationVolumeName = "registration-dir"
pluginDirVolumeName = "plugin-dir"
podsMountDirVolumeName = "pods-mount-dir"
pluginMountDirVolumeName = "plugin-mount-dir"
kmsConfigVolumeName = "ceph-csi-kms-config"
registrationVolumeName = "registration-dir"
pluginDirVolumeName = "plugin-dir"
podsMountDirVolumeName = "pods-mount-dir"
pluginMountDirVolumeName = "plugin-mount-dir"
csiLogDirectoryVolumeName = "csi-log"
csiSidecarLogDirectoryVolumeName = "csi-logs-logrotate"
)

// Ceph CSI common volumes
Expand Down Expand Up @@ -119,6 +121,27 @@ var EtcSelinuxVolume = corev1.Volume{
},
}

func LogDirVolume(LogHostPath, CsiComponentName string) corev1.Volume {
return corev1.Volume{
Name: csiLogDirectoryVolumeName,
VolumeSource: corev1.VolumeSource{
HostPath: &corev1.HostPathVolumeSource{
Path: fmt.Sprintf("%s/log/%s", LogHostPath, CsiComponentName),
Type: ptr.To(corev1.HostPathDirectoryOrCreate),
},
},
}
}

func LogSidecarDirVolume(LogHostPath, CsiComponentName string) corev1.Volume {
return corev1.Volume{
Name: csiSidecarLogDirectoryVolumeName,
VolumeSource: corev1.VolumeSource{
EmptyDir: &corev1.EmptyDirVolumeSource{},
},
}
}

func KmsConfigVolume(configRef *corev1.LocalObjectReference) corev1.Volume {
return corev1.Volume{
VolumeSource: corev1.VolumeSource{
Expand Down Expand Up @@ -234,6 +257,18 @@ var EtcSelinuxVolumeMount = corev1.VolumeMount{
ReadOnly: true,
}

func LogDirVolumeMount(LogHostPath, CsiComponentName string) corev1.VolumeMount {
return corev1.VolumeMount{
Name: csiLogDirectoryVolumeName,
MountPath: fmt.Sprintf("%s/log/%s", LogHostPath, CsiComponentName),
}
}
func LogSidecarVolumeMount(LogHostPath, CsiComponentName string) corev1.VolumeMount {
return corev1.VolumeMount{
Name: csiLogDirectoryVolumeName,
MountPath: fmt.Sprintf("%s/logrotate-config/%s", LogHostPath, CsiComponentName),
}
}
func PodsMountDirVolumeMount(kubletDirPath string) corev1.VolumeMount {
return corev1.VolumeMount{
Name: podsMountDirVolumeName,
Expand Down Expand Up @@ -331,10 +366,15 @@ var TopologyContainerArg = "--feature-gates=Topology=true"
var RecoverVolumeExpansionFailureContainerArg = "--feature-gates=RecoverVolumeExpansionFailure=true"
var EnableVolumeGroupSnapshotsContainerArg = "--enable-volume-group-snapshots=true"
var ForceCephKernelClientContainerArg = "--forcecephkernelclient=true"
var LogToFile = "--logtostderr=false"
var LogToStderr = "--alsologtostderr=true"

func LogLevelContainerArg(level int) string {
return fmt.Sprintf("--v=%d", Clamp(level, 0, 5))
}
func LogRotationSpecArg(LogHostPath, CsiComponentName, containerName string) string {
return fmt.Sprintf("--log_file=%s/log/%s/%s.log", LogHostPath, CsiComponentName, containerName)
}
func TypeContainerArg(t string) string {
switch t {
case "rbd", "cephfs", "nfs", "controller", "liveness":
Expand Down Expand Up @@ -390,3 +430,9 @@ func FuseMountOptionsContainerArg(options map[string]string) string {
"",
)
}

func Privileged() *corev1.SecurityContext {
return &corev1.SecurityContext{
Privileged: ptr.To(true),
}
}

0 comments on commit 10e384a

Please sign in to comment.