diff --git a/alpine.multiarch b/alpine.multiarch index 815a80e2..b6c9820b 100644 --- a/alpine.multiarch +++ b/alpine.multiarch @@ -1,9 +1,17 @@ FROM alpine:3.18.2 +ENV LOGROTATE_ROTATE=5 \ + LOGROTATE_SIZE=100M \ + TZ=Asia/Shanghai + ARG TARGETARCH +ADD hack/logrotate.sh /logrotate.sh +RUN chmod +x /logrotate.sh +RUN ln -sf /usr/share/zoneinfo/$TZ /etc/localtime \ + && echo $TZ > /etc/timezone \ RUN apk update \ && apk upgrade \ && apk add --no-cache \ - curl jq util-linux bash xfsprogs \ + curl jq util-linux bash xfsprogs logrotate \ && rm -rf /var/cache/apk/* diff --git a/apis/apps/v1alpha1/nebulacluster_common.go b/apis/apps/v1alpha1/nebulacluster_common.go index 8901c999..649bd448 100644 --- a/apis/apps/v1alpha1/nebulacluster_common.go +++ b/apis/apps/v1alpha1/nebulacluster_common.go @@ -42,12 +42,13 @@ const ( NebulaServiceAccountName = "nebula-sa" NebulaRoleName = "nebula-role" NebulaRoleBindingName = "nebula-rolebinding" + LogSidecarContainerName = "ng-logrotate" AgentSidecarContainerName = "ng-agent" AgentInitContainerName = "ng-init-agent" DefaultAgentPortGRPC = 8888 - agentPortNameGRPC = "grpc" - defaultAgentImage = "vesoft/nebula-agent" - defaultAlpineImage = "vesoft/nebula-alpine:latest" + AgentPortNameGRPC = "grpc" + DefaultAgentImage = "vesoft/nebula-agent" + DefaultAlpineImage = "vesoft/nebula-alpine:latest" ZoneSuffix = "zone" ) @@ -271,6 +272,16 @@ func parseStorageRequest(res corev1.ResourceList) (corev1.ResourceRequirements, }, nil } +func logVolumeExists(componentType string, volumes []corev1.Volume) bool { + logVolName := logVolume(componentType) + for _, volume := range volumes { + if volume.Name == logVolName { + return true + } + } + return false +} + func GenerateInitAgentContainer(c NebulaClusterComponent) corev1.Container { container := generateAgentContainer(c, true) container.Name = AgentInitContainerName @@ -278,6 +289,43 @@ func GenerateInitAgentContainer(c NebulaClusterComponent) corev1.Container { return container } +func generateLogContainer(c NebulaClusterComponent) corev1.Container { + nc := c.GetNebulaCluster() + componentType := c.ComponentType().String() + + image := DefaultAlpineImage + if nc.Spec.AlpineImage != nil { + image = pointer.StringDeref(nc.Spec.AlpineImage, "") + } + + cmd := []string{"/bin/sh", "-ecx", "sh /logrotate.sh; crond -f -l 2"} + container := corev1.Container{ + Name: LogSidecarContainerName, + Image: image, + Command: cmd, + } + + logRotate := nc.Spec.LogRotate + container.Env = []corev1.EnvVar{ + { + Name: "LOGROTATE_ROTATE", + Value: strconv.Itoa(int(logRotate.Rotate)), + }, + { + Name: "LOGROTATE_SIZE", + Value: logRotate.Size, + }, + } + + container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{ + Name: logVolume(componentType), + MountPath: "/usr/local/nebula/logs", + SubPath: "logs", + }) + + return container +} + func generateAgentContainer(c NebulaClusterComponent, init bool) corev1.Container { nc := c.GetNebulaCluster() componentType := c.ComponentType().String() @@ -288,33 +336,25 @@ func generateAgentContainer(c NebulaClusterComponent, init bool) corev1.Containe fmt.Sprintf(" --agent=$(hostname).%s:%d", c.GetServiceFQDN(), DefaultAgentPortGRPC) + " --ratelimit=1073741824 --debug" brCmd := initCmd + " --meta=" + metadAddr - logCmd := "sh /logrotate.sh; /etc/init.d/cron start" - logfgCmd := "sh /logrotate.sh; exec cron -f" if nc.IsMetadSSLEnabled() || nc.IsClusterSSLEnabled() { - initCmd += " --enable_ssl" - brCmd += " --enable_ssl" + initCmd += " --enable-ssl" + brCmd += " --enable-ssl" if nc.InsecureSkipVerify() { - initCmd += " --insecure_skip_verify" - brCmd += " --insecure_skip_verify" + initCmd += " --insecure-skip-verify" + brCmd += " --insecure-skip-verify" } } if init { cmd = append(cmd, initCmd) } else { - if nc.IsLogRotateEnabled() && nc.IsBREnabled() { - cmd = append(cmd, fmt.Sprintf(`%s; %s`, logCmd, brCmd)) - } else if nc.IsLogRotateEnabled() { - cmd = append(cmd, logfgCmd) - } else if nc.IsBREnabled() { - cmd = append(cmd, brCmd) - } + cmd = append(cmd, brCmd) } container := corev1.Container{ Name: AgentSidecarContainerName, - Image: defaultAgentImage, + Image: DefaultAgentImage, Command: cmd, } imagePullPolicy := nc.Spec.ImagePullPolicy @@ -346,33 +386,13 @@ func generateAgentContainer(c NebulaClusterComponent, init bool) corev1.Containe container.Ports = []corev1.ContainerPort{ { - Name: agentPortNameGRPC, + Name: AgentPortNameGRPC, ContainerPort: int32(DefaultAgentPortGRPC), }, } } - if nc.IsLogRotateEnabled() { - logRotate := nc.Spec.LogRotate - container.Env = []corev1.EnvVar{ - { - Name: "LOGROTATE_ROTATE", - Value: strconv.Itoa(int(logRotate.Rotate)), - }, - { - Name: "LOGROTATE_SIZE", - Value: logRotate.Size, - }, - } - - container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{ - Name: logVolume(componentType), - MountPath: "/usr/local/nebula/logs", - SubPath: "logs", - }) - } - - if (nc.IsMetadSSLEnabled() || nc.IsClusterSSLEnabled()) && nc.IsBREnabled() { + if (nc.IsMetadSSLEnabled() || nc.IsClusterSSLEnabled()) && nc.IsBREnabled() && !enableLocalCerts() { certMounts := []corev1.VolumeMount{ { Name: "client-crt", @@ -412,7 +432,7 @@ NODE_ZONE=$(jq '."topology.kubernetes.io/zone"' -r /node/labels.json) echo "NODE_ZONE is ${NODE_ZONE}" echo "export NODE_ZONE=${NODE_ZONE}" > /node/zone ` - image := defaultAlpineImage + image := DefaultAlpineImage if nc.Spec.AlpineImage != nil { image = pointer.StringDeref(nc.Spec.AlpineImage, "") } @@ -617,10 +637,14 @@ done containers = append(containers, baseContainer) - if nc.IsBREnabled() || nc.IsLogRotateEnabled() { + if nc.IsBREnabled() { agentContainer := generateAgentContainer(c, false) containers = append(containers, agentContainer) } + if nc.IsLogRotateEnabled() && logVolumeExists(componentType, c.GenerateVolumes()) { + logContainer := generateLogContainer(c) + containers = append(containers, logContainer) + } containers = mergeSidecarContainers(containers, c.ComponentSpec().SidecarContainers()) @@ -933,3 +957,9 @@ func separateFlags(config map[string]string) (map[string]string, map[string]stri } return dynamic, static } + +func enableLocalCerts() bool { + return os.Getenv("CA_CERT_PATH") != "" && + os.Getenv("CLIENT_CERT_PATH") != "" && + os.Getenv("CLIENT_KEY_PATH") != "" +} diff --git a/apis/apps/v1alpha1/nebulacluster_types.go b/apis/apps/v1alpha1/nebulacluster_types.go index 710a949b..58141555 100644 --- a/apis/apps/v1alpha1/nebulacluster_types.go +++ b/apis/apps/v1alpha1/nebulacluster_types.go @@ -294,7 +294,6 @@ type ConsoleSpec struct { // AgentContainerSpec defines the desired state of Agent type AgentContainerSpec struct { - // url for docker image // +optional Image string `json:"image,omitempty"` @@ -306,6 +305,16 @@ type AgentContainerSpec struct { // K8S resources settings. // +optional Resources corev1.ResourceRequirements `json:"resources,omitempty"` + + // Container environment variables. + // +optional + EnvVars []corev1.EnvVar `json:"env,omitempty"` + + // +optional + Volumes []corev1.Volume `json:"volumes,omitempty"` + + // +optional + VolumeMounts []corev1.VolumeMount `json:"volumeMounts,omitempty"` } type LicenseSpec struct { diff --git a/hack/logrotate.sh b/hack/logrotate.sh new file mode 100644 index 00000000..974543ec --- /dev/null +++ b/hack/logrotate.sh @@ -0,0 +1,33 @@ +#!/bin/env bash + +ROTATE=5 +SIZE=200M + +if [ -n "${LOGROTATE_ROTATE}" ]; then + ROTATE=${LOGROTATE_ROTATE} +fi + +if [ -n "${LOGROTATE_SIZE}" ]; then + SIZE=${LOGROTATE_SIZE} +fi + +nebula=" +/usr/local/nebula/logs/*.log +/usr/local/nebula/logs/*.impl +/usr/local/nebula/logs/*.INFO +/usr/local/nebula/logs/*.WARNING +/usr/local/nebula/logs/*.ERROR +{ + su root root + daily + rotate ${ROTATE} + copytruncate + nocompress + missingok + notifempty + create 644 root root + size ${SIZE} +} +" + +echo "${nebula}" >/etc/logrotate.d/nebula diff --git a/pkg/controller/component/storaged_updater.go b/pkg/controller/component/storaged_updater.go index 1bb26d26..8dee0e25 100644 --- a/pkg/controller/component/storaged_updater.go +++ b/pkg/controller/component/storaged_updater.go @@ -154,7 +154,7 @@ func (s *storagedUpdater) RestartPod(nc *v1alpha1.NebulaCluster, ordinal int32) updatePod, err := s.clientSet.Pod().GetPod(namespace, updatePodName) if err != nil { - klog.Errorf("get pod failed: %v", namespace, updatePodName, err) + klog.Errorf("get pod [%s/%s] failed: %v", namespace, updatePodName, err) return err } _, ok := updatePod.Annotations[TransLeaderBeginTime] @@ -397,6 +397,7 @@ func (s *storagedUpdater) updateRunningPhase(mc nebula.MetaInterface, nc *v1alph return nil } + // TODO the invoking maybe repeat times for _, space := range spaces { if err := mc.BalanceLeader(*space.Id.SpaceID); err != nil { return err