From 38d8450eb98a87261ed5bbb6d01c6fa0ef30574b Mon Sep 17 00:00:00 2001 From: cyclinder Date: Thu, 25 Jul 2024 18:53:38 +0800 Subject: [PATCH] spiderpool-agent: support to configure the sysctl config Signed-off-by: cyclinder --- charts/spiderpool/README.md | 107 +++++++++++---------- charts/spiderpool/templates/daemonset.yaml | 5 +- charts/spiderpool/values.yaml | 3 + cmd/spiderpool-agent/cmd/daemon.go | 33 +++++++ docs/reference/spiderpool-agent.md | 23 +++++ pkg/networking/sysctl/sysctl.go | 63 +++++++++++- test/scripts/install-multus.sh | 3 +- 7 files changed, 181 insertions(+), 56 deletions(-) diff --git a/charts/spiderpool/README.md b/charts/spiderpool/README.md index bd13460786..67b388d15c 100644 --- a/charts/spiderpool/README.md +++ b/charts/spiderpool/README.md @@ -242,59 +242,60 @@ helm install spiderpool spiderpool/spiderpool --wait --namespace kube-system \ ### spiderpoolAgent parameters -| Name | Description | Value | -| ------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------ | ------------------------------------------ | -| `spiderpoolAgent.name` | the spiderpoolAgent name | `spiderpool-agent` | -| `spiderpoolAgent.binName` | the binName name of spiderpoolAgent | `/usr/bin/spiderpool-agent` | -| `spiderpoolAgent.image.registry` | the image registry of spiderpoolAgent | `ghcr.io` | -| `spiderpoolAgent.image.repository` | the image repository of spiderpoolAgent | `spidernet-io/spiderpool/spiderpool-agent` | -| `spiderpoolAgent.image.pullPolicy` | the image pullPolicy of spiderpoolAgent | `IfNotPresent` | -| `spiderpoolAgent.image.digest` | the image digest of spiderpoolAgent, which takes preference over tag | `""` | -| `spiderpoolAgent.image.tag` | the image tag of spiderpoolAgent, overrides the image tag whose default is the chart appVersion. | `""` | -| `spiderpoolAgent.image.imagePullSecrets` | the image imagePullSecrets of spiderpoolAgent | `[]` | -| `spiderpoolAgent.serviceAccount.create` | create the service account for the spiderpoolAgent | `true` | -| `spiderpoolAgent.serviceAccount.annotations` | the annotations of spiderpoolAgent service account | `{}` | -| `spiderpoolAgent.service.annotations` | the annotations for spiderpoolAgent service | `{}` | -| `spiderpoolAgent.service.type` | the type for spiderpoolAgent service | `ClusterIP` | -| `spiderpoolAgent.priorityClassName` | the priority Class Name for spiderpoolAgent | `system-node-critical` | -| `spiderpoolAgent.affinity` | the affinity of spiderpoolAgent | `{}` | -| `spiderpoolAgent.extraArgs` | the additional arguments of spiderpoolAgent container | `[]` | -| `spiderpoolAgent.extraEnv` | the additional environment variables of spiderpoolAgent container | `[]` | -| `spiderpoolAgent.extraVolumes` | the additional volumes of spiderpoolAgent container | `[]` | -| `spiderpoolAgent.extraVolumeMounts` | the additional hostPath mounts of spiderpoolAgent container | `[]` | -| `spiderpoolAgent.podAnnotations` | the additional annotations of spiderpoolAgent pod | `{}` | -| `spiderpoolAgent.podLabels` | the additional label of spiderpoolAgent pod | `{}` | -| `spiderpoolAgent.resources.limits.cpu` | the cpu limit of spiderpoolAgent pod | `1000m` | -| `spiderpoolAgent.resources.limits.memory` | the memory limit of spiderpoolAgent pod | `1024Mi` | -| `spiderpoolAgent.resources.requests.cpu` | the cpu requests of spiderpoolAgent pod | `100m` | -| `spiderpoolAgent.resources.requests.memory` | the memory requests of spiderpoolAgent pod | `128Mi` | -| `spiderpoolAgent.securityContext` | the security Context of spiderpoolAgent pod | `{}` | -| `spiderpoolAgent.httpPort` | the http Port for spiderpoolAgent, for health checking | `5710` | -| `spiderpoolAgent.healthChecking.startupProbe.failureThreshold` | the failure threshold of startup probe for spiderpoolAgent health checking | `60` | -| `spiderpoolAgent.healthChecking.startupProbe.periodSeconds` | the period seconds of startup probe for spiderpoolAgent health checking | `2` | -| `spiderpoolAgent.healthChecking.livenessProbe.failureThreshold` | the failure threshold of startup probe for spiderpoolAgent health checking | `6` | -| `spiderpoolAgent.healthChecking.livenessProbe.periodSeconds` | the period seconds of startup probe for spiderpoolAgent health checking | `10` | -| `spiderpoolAgent.healthChecking.readinessProbe.failureThreshold` | the failure threshold of startup probe for spiderpoolAgent health checking | `3` | -| `spiderpoolAgent.healthChecking.readinessProbe.periodSeconds` | the period seconds of startup probe for spiderpoolAgent health checking | `10` | -| `spiderpoolAgent.prometheus.enabled` | enable spiderpool agent to collect metrics | `false` | -| `spiderpoolAgent.prometheus.enabledDebugMetric` | enable spiderpool agent to collect debug level metrics | `false` | -| `spiderpoolAgent.prometheus.port` | the metrics port of spiderpool agent | `5711` | -| `spiderpoolAgent.prometheus.serviceMonitor.install` | install serviceMonitor for spiderpool agent. This requires the prometheus CRDs to be available | `false` | -| `spiderpoolAgent.prometheus.serviceMonitor.namespace` | the serviceMonitor namespace. Default to the namespace of helm instance | `""` | -| `spiderpoolAgent.prometheus.serviceMonitor.annotations` | the additional annotations of spiderpoolAgent serviceMonitor | `{}` | -| `spiderpoolAgent.prometheus.serviceMonitor.labels` | the additional label of spiderpoolAgent serviceMonitor | `{}` | -| `spiderpoolAgent.prometheus.serviceMonitor.interval` | represents the interval of spiderpoolAgent serviceMonitor's scraping action | `10s` | -| `spiderpoolAgent.prometheus.prometheusRule.install` | install prometheusRule for spiderpool agent. This requires the prometheus CRDs to be available | `false` | -| `spiderpoolAgent.prometheus.prometheusRule.namespace` | the prometheusRule namespace. Default to the namespace of helm instance | `""` | -| `spiderpoolAgent.prometheus.prometheusRule.annotations` | the additional annotations of spiderpoolAgent prometheusRule | `{}` | -| `spiderpoolAgent.prometheus.prometheusRule.labels` | the additional label of spiderpoolAgent prometheusRule | `{}` | -| `spiderpoolAgent.prometheus.prometheusRule.enableWarningIPAMAllocationFailure` | the additional rule of spiderpoolAgent prometheusRule | `true` | -| `spiderpoolAgent.prometheus.prometheusRule.enableWarningIPAMAllocationOverTime` | the additional rule of spiderpoolAgent prometheusRule | `true` | -| `spiderpoolAgent.prometheus.prometheusRule.enableWarningIPAMHighAllocationDurations` | the additional rule of spiderpoolAgent prometheusRule | `true` | -| `spiderpoolAgent.prometheus.prometheusRule.enableWarningIPAMReleaseFailure` | the additional rule of spiderpoolAgent prometheusRule | `true` | -| `spiderpoolAgent.prometheus.prometheusRule.enableWarningIPAMReleaseOverTime` | the additional rule of spiderpoolAgent prometheusRule | `true` | -| `spiderpoolAgent.debug.logLevel` | the log level of spiderpool agent [debug, info, warn, error, fatal, panic] | `info` | -| `spiderpoolAgent.debug.gopsPort` | the gops port of spiderpool agent | `5712` | +| Name | Description | Value | +| ------------------------------------------------------------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------ | +| `spiderpoolAgent.name` | the spiderpoolAgent name | `spiderpool-agent` | +| `spiderpoolAgent.binName` | the binName name of spiderpoolAgent | `/usr/bin/spiderpool-agent` | +| `spiderpoolAgent.image.registry` | the image registry of spiderpoolAgent | `ghcr.io` | +| `spiderpoolAgent.image.repository` | the image repository of spiderpoolAgent | `spidernet-io/spiderpool/spiderpool-agent` | +| `spiderpoolAgent.image.pullPolicy` | the image pullPolicy of spiderpoolAgent | `IfNotPresent` | +| `spiderpoolAgent.image.digest` | the image digest of spiderpoolAgent, which takes preference over tag | `""` | +| `spiderpoolAgent.image.tag` | the image tag of spiderpoolAgent, overrides the image tag whose default is the chart appVersion. | `""` | +| `spiderpoolAgent.image.imagePullSecrets` | the image imagePullSecrets of spiderpoolAgent | `[]` | +| `spiderpoolAgent.serviceAccount.create` | create the service account for the spiderpoolAgent | `true` | +| `spiderpoolAgent.serviceAccount.annotations` | the annotations of spiderpoolAgent service account | `{}` | +| `spiderpoolAgent.service.annotations` | the annotations for spiderpoolAgent service | `{}` | +| `spiderpoolAgent.service.type` | the type for spiderpoolAgent service | `ClusterIP` | +| `spiderpoolAgent.priorityClassName` | the priority Class Name for spiderpoolAgent | `system-node-critical` | +| `spiderpoolAgent.affinity` | the affinity of spiderpoolAgent | `{}` | +| `spiderpoolAgent.extraArgs` | the additional arguments of spiderpoolAgent container | `[]` | +| `spiderpoolAgent.extraEnv` | the additional environment variables of spiderpoolAgent container | `[]` | +| `spiderpoolAgent.extraVolumes` | the additional volumes of spiderpoolAgent container | `[]` | +| `spiderpoolAgent.extraVolumeMounts` | the additional hostPath mounts of spiderpoolAgent container | `[]` | +| `spiderpoolAgent.podAnnotations` | the additional annotations of spiderpoolAgent pod | `{}` | +| `spiderpoolAgent.podLabels` | the additional label of spiderpoolAgent pod | `{}` | +| `spiderpoolAgent.resources.limits.cpu` | the cpu limit of spiderpoolAgent pod | `1000m` | +| `spiderpoolAgent.resources.limits.memory` | the memory limit of spiderpoolAgent pod | `1024Mi` | +| `spiderpoolAgent.resources.requests.cpu` | the cpu requests of spiderpoolAgent pod | `100m` | +| `spiderpoolAgent.resources.requests.memory` | the memory requests of spiderpoolAgent pod | `128Mi` | +| `spiderpoolAgent.tuneSysctlConfig` | enable to set required sysctl on each node to run spiderpool. refer to [Spiderpool-agent](https://spidernet-io.github.io/spiderpool/dev/reference/spiderpool-agent/) for details | `true` | +| `spiderpoolAgent.securityContext` | the security Context of spiderpoolAgent pod | `{}` | +| `spiderpoolAgent.httpPort` | the http Port for spiderpoolAgent, for health checking | `5710` | +| `spiderpoolAgent.healthChecking.startupProbe.failureThreshold` | the failure threshold of startup probe for spiderpoolAgent health checking | `60` | +| `spiderpoolAgent.healthChecking.startupProbe.periodSeconds` | the period seconds of startup probe for spiderpoolAgent health checking | `2` | +| `spiderpoolAgent.healthChecking.livenessProbe.failureThreshold` | the failure threshold of startup probe for spiderpoolAgent health checking | `6` | +| `spiderpoolAgent.healthChecking.livenessProbe.periodSeconds` | the period seconds of startup probe for spiderpoolAgent health checking | `10` | +| `spiderpoolAgent.healthChecking.readinessProbe.failureThreshold` | the failure threshold of startup probe for spiderpoolAgent health checking | `3` | +| `spiderpoolAgent.healthChecking.readinessProbe.periodSeconds` | the period seconds of startup probe for spiderpoolAgent health checking | `10` | +| `spiderpoolAgent.prometheus.enabled` | enable spiderpool agent to collect metrics | `false` | +| `spiderpoolAgent.prometheus.enabledDebugMetric` | enable spiderpool agent to collect debug level metrics | `false` | +| `spiderpoolAgent.prometheus.port` | the metrics port of spiderpool agent | `5711` | +| `spiderpoolAgent.prometheus.serviceMonitor.install` | install serviceMonitor for spiderpool agent. This requires the prometheus CRDs to be available | `false` | +| `spiderpoolAgent.prometheus.serviceMonitor.namespace` | the serviceMonitor namespace. Default to the namespace of helm instance | `""` | +| `spiderpoolAgent.prometheus.serviceMonitor.annotations` | the additional annotations of spiderpoolAgent serviceMonitor | `{}` | +| `spiderpoolAgent.prometheus.serviceMonitor.labels` | the additional label of spiderpoolAgent serviceMonitor | `{}` | +| `spiderpoolAgent.prometheus.serviceMonitor.interval` | represents the interval of spiderpoolAgent serviceMonitor's scraping action | `10s` | +| `spiderpoolAgent.prometheus.prometheusRule.install` | install prometheusRule for spiderpool agent. This requires the prometheus CRDs to be available | `false` | +| `spiderpoolAgent.prometheus.prometheusRule.namespace` | the prometheusRule namespace. Default to the namespace of helm instance | `""` | +| `spiderpoolAgent.prometheus.prometheusRule.annotations` | the additional annotations of spiderpoolAgent prometheusRule | `{}` | +| `spiderpoolAgent.prometheus.prometheusRule.labels` | the additional label of spiderpoolAgent prometheusRule | `{}` | +| `spiderpoolAgent.prometheus.prometheusRule.enableWarningIPAMAllocationFailure` | the additional rule of spiderpoolAgent prometheusRule | `true` | +| `spiderpoolAgent.prometheus.prometheusRule.enableWarningIPAMAllocationOverTime` | the additional rule of spiderpoolAgent prometheusRule | `true` | +| `spiderpoolAgent.prometheus.prometheusRule.enableWarningIPAMHighAllocationDurations` | the additional rule of spiderpoolAgent prometheusRule | `true` | +| `spiderpoolAgent.prometheus.prometheusRule.enableWarningIPAMReleaseFailure` | the additional rule of spiderpoolAgent prometheusRule | `true` | +| `spiderpoolAgent.prometheus.prometheusRule.enableWarningIPAMReleaseOverTime` | the additional rule of spiderpoolAgent prometheusRule | `true` | +| `spiderpoolAgent.debug.logLevel` | the log level of spiderpool agent [debug, info, warn, error, fatal, panic] | `info` | +| `spiderpoolAgent.debug.gopsPort` | the gops port of spiderpool agent | `5712` | ### spiderpoolController parameters diff --git a/charts/spiderpool/templates/daemonset.yaml b/charts/spiderpool/templates/daemonset.yaml index c05a56a991..7bb11a9089 100644 --- a/charts/spiderpool/templates/daemonset.yaml +++ b/charts/spiderpool/templates/daemonset.yaml @@ -197,10 +197,13 @@ spec: {{- with .Values.spiderpoolAgent.extraEnv }} {{- toYaml . | nindent 8 }} {{- end }} - {{- with .Values.spiderpoolAgent.securityContext }} + {{- if or .Values.spiderpoolAgent.tuneSysctlConfig .Values.spiderpoolAgent.securityContext }} securityContext: + privileged: true + {{- with .Values.spiderpoolAgent.securityContext }} {{- toYaml . | nindent 8 }} {{- end }} + {{- end }} volumeMounts: - name: config-path mountPath: /tmp/spiderpool/config-map diff --git a/charts/spiderpool/values.yaml b/charts/spiderpool/values.yaml index 8857d91f17..38c097781c 100644 --- a/charts/spiderpool/values.yaml +++ b/charts/spiderpool/values.yaml @@ -420,6 +420,9 @@ spiderpoolAgent: ## @param spiderpoolAgent.resources.requests.memory the memory requests of spiderpoolAgent pod memory: 128Mi + ## @param spiderpoolAgent.tuneSysctlConfig enable to set required sysctl on each node to run spiderpool. refer to [Spiderpool-agent](https://spidernet-io.github.io/spiderpool/dev/reference/spiderpool-agent/) for details + tuneSysctlConfig: true + ## @param spiderpoolAgent.securityContext the security Context of spiderpoolAgent pod securityContext: {} # runAsUser: 0 diff --git a/cmd/spiderpool-agent/cmd/daemon.go b/cmd/spiderpool-agent/cmd/daemon.go index ca5b8cd55e..9359e57ed2 100644 --- a/cmd/spiderpool-agent/cmd/daemon.go +++ b/cmd/spiderpool-agent/cmd/daemon.go @@ -27,6 +27,7 @@ import ( "github.com/spidernet-io/spiderpool/pkg/kubevirtmanager" "github.com/spidernet-io/spiderpool/pkg/logutils" "github.com/spidernet-io/spiderpool/pkg/namespacemanager" + "github.com/spidernet-io/spiderpool/pkg/networking/sysctl" "github.com/spidernet-io/spiderpool/pkg/nodemanager" "github.com/spidernet-io/spiderpool/pkg/openapi" "github.com/spidernet-io/spiderpool/pkg/podmanager" @@ -73,6 +74,15 @@ func DaemonMain() { } logger.Sugar().Infof("Spiderpool-agent config: %+v", agentContext.Cfg) + // setup sysctls + if agentContext.Cfg.TuneSysctlConfig { + if err := sysctlConfig(agentContext.Cfg.EnableIPv4, agentContext.Cfg.EnableIPv6); err != nil { + logger.Sugar().Fatal(err) + } + } else { + logger.Sugar().Infof("setSysctlConfig is disabled.") + } + // Set up gops. if agentContext.Cfg.GopsListenPort != "" { address := "127.0.0.1:" + agentContext.Cfg.GopsListenPort @@ -411,3 +421,26 @@ func initAgentServiceManagers(ctx context.Context) { logger.Info("Feature SpiderSubnet is disabled") } } + +// sysctlConfig set default sysctl configs,Notice: ignore not exist sysctl configs as +// possible. +func sysctlConfig(enableIPv4, enableIPv6 bool) error { + // setup default sysctl config + for _, sc := range sysctl.DefaultSysctlConfig { + if (enableIPv4 && sc.IsIPv4) || (enableIPv6 && sc.IsIPv6) { + logger.Info("Setup sysctl", zap.String("sysctl", sc.Name), zap.String("value", sc.Value)) + err := sysctl.SetSysctl(sc.Name, sc.Value) + if err == nil { + logger.Debug("success to setup sysctl", zap.String("sysctl", sc.Name), zap.String("value", sc.Value)) + continue + } + + if !errors.Is(err, os.ErrNotExist) { + logger.Error("failed to setup sysctl", zap.String("sysctl", sc.Name), zap.String("value", sc.Value), zap.Error(err)) + return err + } + logger.Warn("skip to setup sysctl", zap.String("sysctl", sc.Name), zap.String("value", sc.Value), zap.Error(err)) + } + } + return nil +} diff --git a/docs/reference/spiderpool-agent.md b/docs/reference/spiderpool-agent.md index af9bfb848d..253439e4bc 100644 --- a/docs/reference/spiderpool-agent.md +++ b/docs/reference/spiderpool-agent.md @@ -26,6 +26,29 @@ Run the spiderpool agent daemon. | SPIDERPOOL_WORKLOADENDPOINT_MAX_HISTORY_RECORDS | 100 | Max historical IP allocation information allowed for a single Pod recorded in WorkloadEndpoint. | | SPIDERPOOL_IPPOOL_MAX_ALLOCATED_IPS | 5000 | Max number of IP that a single IP pool can provide. | +## spiderpool-agent helps set sysctl configs for each node + +To optimize the kernel network configuration of a node, spiderpool-agent will by default configure the following kernel parameters: + +| sysctl config | value | description | +| -------------| ------| ------------| +| net.ipv4.neigh.default.gc_thresh3 | 28160 | This is the hard maximum number of entries to keep in the ARP cache. The garbage collector will always run if there are more than this number of entries in the cache. for ipv4 | +| net.ipv6.neigh.default.gc_thresh3 | 28160 | This is the hard maximum number of entries to keep in the ARP cache. The garbage collector will always run if there are more than this number of entries in the cache. for ipv6. Note: this is only avaliable in some low kernel version.| +| net.ipv4.conf.all.arp_notify | 1 | Generate gratuitous arp requests when device is brought up or hardware address changes.| +| net.ipv4.conf.all.forwarding | 1 | enable ipv4 forwarding | +| net.ipv4.conf.all.forwarding | 1 | enable ipv6 forwarding | + +To optimize the kernel network configuration of a node, spiderpool-agent configures some kernel parameters (such as a, etc.) by default. Some kernel parameters can only be set in certain kernel versions, so we will ignore the "kernel parameter does not exist" error when configure the kernel parameters. + +Users can edit the `spiderpoolAgent.securityContext` field of values.yaml in the chart before installing spiderpool to update the kernel parameters that need additional configuration, or manually edit spiderpool-agent daemonSet after installing Spiderpool, and then restart spiderpool-agent pods: + +Users can disable this feature by following command when installing Spiderpool: + +``` +helm install spiderpool -n kube-system --set global.tuneSysctlConfig=false +``` + +Or configure the spiderpool-conf configMap, set tuneSysctlConfig to false and restart the spiderpool-agent pods. ## spiderpool-agent shutdown diff --git a/pkg/networking/sysctl/sysctl.go b/pkg/networking/sysctl/sysctl.go index ea50734d4d..88f6ebe0a0 100644 --- a/pkg/networking/sysctl/sysctl.go +++ b/pkg/networking/sysctl/sysctl.go @@ -5,11 +5,55 @@ package sysctl import ( "fmt" + "os" + "path/filepath" + "strings" + "github.com/containernetworking/plugins/pkg/ns" "github.com/containernetworking/plugins/pkg/utils/sysctl" - "os" ) +// DefaultSysctlConfig is the default sysctl config for the node +var DefaultSysctlConfig = []struct { + Name string + Value string + IsIPv4, IsIPv6 bool +}{ + // In order to avoid large-scale cluster arp_table overflow, resulting in + // pods not being able to communicate or pods not being able to start due + // to the inability to insert static arp table entries, it is necessary + // to appropriately increase and adjust its value. more details see: + // https://github.com/spidernet-io/spiderpool/issues/3587 + { + Name: "net.ipv4.neigh.default.gc_thresh3", + // Assuming a node is full of underlay pods (110) and their subnet + // mask is 16 bits ( 2 ^ 8 = 256 IPs), the value is 110 * 256 = 28160 + Value: "28160", + IsIPv4: true, + }, + { + // this sysctl may not be available at low kernel levels, + // so we'll ignore it at this point. + Name: "net.ipv6.neigh.default.gc_thresh3", + Value: "28160", + IsIPv6: true, + }, + // send gratitous ARP when device or address change + { + Name: "net.ipv4.conf.all.arp_notify", + Value: "1", + IsIPv4: true, + }, { + Name: "net.ipv4.conf.all.forwarding", + Value: "1", + IsIPv4: true, + }, { + Name: "net.ipv6.conf.all.forwarding", + Value: "1", + IsIPv6: true, + }, +} + // SysctlRPFilter set rp_filter value for host netns and specify netns func SysctlRPFilter(netns ns.NetNS, value int32) error { var err error @@ -77,3 +121,20 @@ func EnableIpv6Sysctl(netns ns.NetNS) error { }) return err } + +func SetSysctl(sysConfig string, value string) error { + // sysConfig: net.ipv6.neigh.default.gc_thresh3 + // to: net/ipv6/neigh/default/gc_thresh3 + sysConfig = strings.ReplaceAll(sysConfig, ".", "/") + + _, err := os.Stat(filepath.Join("/proc/sys", sysConfig)) + if err != nil { + return err + } + + if _, err := sysctl.Sysctl(sysConfig, value); err != nil { + return err + } + + return nil +} diff --git a/test/scripts/install-multus.sh b/test/scripts/install-multus.sh index a763797268..6eebc0e9e2 100755 --- a/test/scripts/install-multus.sh +++ b/test/scripts/install-multus.sh @@ -377,7 +377,8 @@ EOF kubectl wait --for=condition=ready -l app.kubernetes.io/component=spiderpool-agent --timeout=100s pod -n kube-system --kubeconfig ${E2E_KUBECONFIG} || \ ( kubectl get pod -n kube-system --kubeconfig ${E2E_KUBECONFIG} ; \ - kubectl logs -n kube-system -l job-name=spiderpool-init --kubeconfig ${E2E_KUBECONFIG} ; exit 1 ) + kubectl logs -n kube-system -l app.kubernetes.io/component=spiderpool-agent --kubeconfig ${E2E_KUBECONFIG} ; \ + kubectl logs -n kube-system -l job-name=spiderpool-init --kubeconfig ${E2E_KUBECONFIG} ; exit 1 ) Install::MultusCR Install::SpiderpoolCR