diff --git a/pkg/kube/cluster-init.sh b/pkg/kube/cluster-init.sh index 46c1fcf4744..bc8e3c47f2d 100755 --- a/pkg/kube/cluster-init.sh +++ b/pkg/kube/cluster-init.sh @@ -187,6 +187,54 @@ trigger_k3s_selfextraction() { #Make sure all prereqs are set after /var/lib is mounted to get logging info setup_prereqs +VMICONFIG_FILENAME="/run/zedkube/vmiVNC.run" +VNC_RUNNING=false +# run virtctl vnc +check_and_run_vnc() { + pid=$(pgrep -f "/usr/bin/virtctl vnc" ) + # if remote-console config file exist, and either has not started, or need to restart + if [ -f "$VMICONFIG_FILENAME" ] && { [ "$VNC_RUNNING" = false ] || [ -z "$pid" ]; } then + vmiName="" + vmiPort="" + + # Read the file and extract values + while IFS= read -r line; do + case "$line" in + *"VMINAME:"*) + vmiName="${line#*VMINAME:}" # Extract the part after "VMINAME:" + vmiName="${vmiName%%[[:space:]]*}" # Remove leading/trailing whitespace + ;; + *"VNCPORT:"*) + vmiPort="${line#*VNCPORT:}" # Extract the part after "VNCPORT:" + vmiPort="${vmiPort%%[[:space:]]*}" # Remove leading/trailing whitespace + ;; + esac + done < "$VMICONFIG_FILENAME" + + # Check if vminame and vncport were found and assign default values if not + if [ -z "$vmiName" ] || [ -z "$vmiPort" ]; then + logmsg "Error: VMINAME or VNCPORT is empty in $VMICONFIG_FILENAME" + return 1 + fi + + logmsg "virctl vnc on vmiName: $vmiName, port $vmiPort" + nohup /usr/bin/virtctl vnc "$vmiName" -n eve-kube-app --port "$vmiPort" --proxy-only & + VNC_RUNNING=true + else + if [ ! -f "$VMICONFIG_FILENAME" ]; then + if [ "$VNC_RUNNING" = true ]; then + if [ -n "$pid" ]; then + logmsg "Killing process with PID $pid" + kill -9 "$pid" + else + logmsg "Error: Process not found" + fi + fi + VNC_RUNNING=false + fi + fi +} + date >> $INSTALL_LOG #Forever loop every 15 secs @@ -293,5 +341,8 @@ fi cp "$CTRD_LOG" "${CTRD_LOG}.1" truncate -s 0 "$CTRD_LOG" fi + + # Check and run vnc + check_and_run_vnc sleep 15 done diff --git a/pkg/pillar/cmd/zedkube/applogs.go b/pkg/pillar/cmd/zedkube/applogs.go new file mode 100644 index 00000000000..7f6d12e2b59 --- /dev/null +++ b/pkg/pillar/cmd/zedkube/applogs.go @@ -0,0 +1,100 @@ +// Copyright (c) 2024 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +//go:build kubevirt + +package zedkube + +import ( + "bufio" + "context" + "io" + "regexp" + "strings" + "time" + + "github.com/lf-edge/eve/pkg/pillar/base" + "github.com/lf-edge/eve/pkg/pillar/kubeapi" + "github.com/lf-edge/eve/pkg/pillar/types" + "github.com/sirupsen/logrus" + corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/kubernetes" +) + +// collectAppLogs - collect App logs from pods which covers both containers and virt-launcher pods +func collectAppLogs(ctx *zedkubeContext) { + sub := ctx.subAppInstanceConfig + items := sub.GetAll() + if len(items) == 0 { + return + } + + if ctx.config == nil { + config, err := kubeapi.GetKubeConfig() + if err != nil { + return + } + ctx.config = config + } + clientset, err := kubernetes.NewForConfig(ctx.config) + if err != nil { + log.Errorf("collectAppLogs: can't get clientset %v", err) + return + } + + // "Thu Aug 17 05:39:04 UTC 2023" + timestampRegex := regexp.MustCompile(`(\w{3} \w{3} \d{2} \d{2}:\d{2}:\d{2} \w+ \d{4})`) + nowStr := time.Now().String() + + var sinceSec int64 + sinceSec = logcollectInterval + for _, item := range items { + aiconfig := item.(types.AppInstanceConfig) + contName := base.GetAppKubeName(aiconfig.DisplayName, aiconfig.UUIDandVersion.UUID) + + opt := &corev1.PodLogOptions{} + if ctx.appLogStarted { + opt = &corev1.PodLogOptions{ + SinceSeconds: &sinceSec, + } + } else { + ctx.appLogStarted = true + } + req := clientset.CoreV1().Pods(kubeapi.EVEKubeNameSpace).GetLogs(contName, opt) + podLogs, err := req.Stream(context.Background()) + if err != nil { + log.Errorf("collectAppLogs: pod %s, log error %v", contName, err) + continue + } + defer podLogs.Close() + + scanner := bufio.NewScanner(podLogs) + for scanner.Scan() { + logLine := scanner.Text() + + matches := timestampRegex.FindStringSubmatch(logLine) + var timeStr string + if len(matches) > 0 { + timeStr = matches[0] + ts := strings.Split(logLine, timeStr) + if len(ts) > 1 { + logLine = ts[0] + } + } else { + timeStr = nowStr + } + // Process and print the log line here + aiLogger := ctx.appContainerLogger.WithFields(logrus.Fields{ + "appuuid": aiconfig.UUIDandVersion.UUID.String(), + "containername": contName, + "eventtime": timeStr, + }) + aiLogger.Infof("%s", logLine) + } + if scanner.Err() != nil { + if scanner.Err() == io.EOF { + break // Break out of the loop when EOF is reached + } + } + } +} diff --git a/pkg/pillar/cmd/zedkube/etherpassthrough.go b/pkg/pillar/cmd/zedkube/etherpassthrough.go new file mode 100644 index 00000000000..7d1a8217da4 --- /dev/null +++ b/pkg/pillar/cmd/zedkube/etherpassthrough.go @@ -0,0 +1,99 @@ +// Copyright (c) 2024 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +//go:build kubevirt + +package zedkube + +import ( + "fmt" + + "github.com/lf-edge/eve/pkg/pillar/kubeapi" + "github.com/lf-edge/eve/pkg/pillar/types" + "github.com/vishvananda/netlink" +) + +// checkIoAdapterEthernet - check and create NAD for direct-attached ethernet +func checkIoAdapterEthernet(ctx *zedkubeContext, aiConfig *types.AppInstanceConfig) error { + + if aiConfig.FixedResources.VirtualizationMode != types.NOHYPER { + return nil + } + ioAdapter := aiConfig.IoAdapterList + for _, io := range ioAdapter { + if io.Type == types.IoNetEth { + nadname := "host-" + io.Name + _, ok := ctx.networkInstanceStatusMap.Load(nadname) + if !ok { + bringupInterface(io.Name) + err := ioEtherCreate(ctx, &io) + if err != nil { + log.Errorf("checkIoAdapterEthernet: create io adapter error %v", err) + } + ctx.ioAdapterMap.Store(nadname, true) + log.Functionf("ccheckIoAdapterEthernet: nad created %v", nadname) + } else { + log.Functionf("checkIoAdapterEthernet: nad already exist %v", nadname) + } + } + } + return nil +} + +func checkDelIoAdpaterEthernet(ctx *zedkubeContext, aiConfig *types.AppInstanceConfig) { + + if aiConfig.FixedResources.VirtualizationMode != types.NOHYPER { + return + } + ioAdapter := aiConfig.IoAdapterList + for _, io := range ioAdapter { + if io.Type == types.IoNetEth { + nadname := "host-" + io.Name + _, ok := ctx.ioAdapterMap.Load(nadname) + if ok { + // remove the syncMap entry + ctx.ioAdapterMap.Delete(nadname) + } + // delete the NAD in kubernetes + kubeapi.DeleteNAD(log, nadname) + log.Functionf("checkDelIoAdpaterEthernet: delete existing nad %v", nadname) + } + } +} + +// ioEtherCreate - create and send NAD for direct-attached ethernet +func ioEtherCreate(ctx *zedkubeContext, ioAdapt *types.IoAdapter) error { + name := ioAdapt.Name + spec := fmt.Sprintf( + `{ + "cniVersion": "0.3.1", + "plugins": [ + { + "type": "host-device", + "device": "%s" + } + ] +}`, name) + + err := kubeapi.CreateOrUpdateNAD(log, "host-"+name, spec) + if err != nil { + log.Errorf("ioEtherCreate: spec, CreateOrUpdateNAD, error %v", err) + } else { + log.Functionf("ioEtherCreate: spec, CreateOrUpdateNAD, done") + } + return err +} + +func bringupInterface(intfName string) { + link, err := netlink.LinkByName(intfName) + if err != nil { + log.Errorf("bringupInterface: %v", err) + return + } + + // Set the IFF_UP flag to bring up the interface + if err := netlink.LinkSetUp(link); err != nil { + log.Errorf("bringupInterface: %v", err) + return + } +} diff --git a/pkg/pillar/cmd/zedkube/nokube.go b/pkg/pillar/cmd/zedkube/nokube.go new file mode 100644 index 00000000000..d1edea53ce0 --- /dev/null +++ b/pkg/pillar/cmd/zedkube/nokube.go @@ -0,0 +1,45 @@ +// Copyright (c) 2024 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +//go:build !kubevirt + +package zedkube + +import ( + "time" + + "github.com/lf-edge/eve/pkg/pillar/agentbase" + "github.com/lf-edge/eve/pkg/pillar/base" + "github.com/lf-edge/eve/pkg/pillar/pubsub" + "github.com/sirupsen/logrus" +) + +const ( + agentName = "zedkube" + // Time limits for event loop handlers + errorTime = 3 * time.Minute + warningTime = 40 * time.Second + stillRunningInterval = 25 * time.Second +) + +type zedkubeContext struct { + agentbase.AgentBase +} + +// Run in this file is just stub for non-kubevirt hypervisors. +func Run(ps *pubsub.PubSub, loggerArg *logrus.Logger, logArg *base.LogObject, arguments []string) int { + logger := loggerArg + log := logArg + + zedkubeCtx := zedkubeContext{} + agentbase.Init(&zedkubeCtx, logger, log, agentName, + agentbase.WithPidFile(), + agentbase.WithWatchdog(ps, warningTime, errorTime), + agentbase.WithArguments(arguments)) + + stillRunning := time.NewTicker(stillRunningInterval) + for range stillRunning.C { + ps.StillRunning(agentName, warningTime, errorTime) + } + return 0 +} diff --git a/pkg/pillar/cmd/zedkube/vnc.go b/pkg/pillar/cmd/zedkube/vnc.go new file mode 100644 index 00000000000..406d0a0d7d4 --- /dev/null +++ b/pkg/pillar/cmd/zedkube/vnc.go @@ -0,0 +1,94 @@ +// Copyright (c) 2024 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +//go:build kubevirt + +package zedkube + +import ( + "context" + "fmt" + "os" + "strconv" + "strings" + "time" + + "github.com/lf-edge/eve/pkg/pillar/base" + "github.com/lf-edge/eve/pkg/pillar/kubeapi" + "github.com/lf-edge/eve/pkg/pillar/types" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// runAppVNC - run vnc for kubevirt VMI remote console +func runAppVNC(ctx *zedkubeContext, config *types.AppInstanceConfig) { + vmconfig := config.FixedResources + + //vmiName := findXenCfgName(config.UUIDandVersion.UUID.String()) + var vmiName string + i := 5 + for { + var err error + vmiName, err = getVMIdomainName(ctx, config) + if err != nil { + log.Functionf("runAppVNC: get vmi domainname error %v", err) + if i >= 0 { + time.Sleep(3 * time.Second) + continue + } + } else { + break + } + i = i - 1 + } + if vmiName == "" { + log.Functionf("runAppVNC: can not find vmiName") + return + } + vncPort := vmconfig.VncDisplay + 5900 + port := strconv.Itoa(int(vncPort)) + + if config.RemoteConsole { + content := fmt.Sprintf("VMINAME:%s\nVNCPORT:%s\n", vmiName, port) + err := os.WriteFile(vmiVNCFileName, []byte(content), 0644) + if err != nil { + log.Errorf("runAppVNC: Error creating file: %v", err) + return + } + log.Functionf("runAppVNC: vmiName %s, port %s, vmiVNC file created", vmiName, port) + } else { + if _, err := os.Stat(vmiVNCFileName); err == nil { + err = os.Remove(vmiVNCFileName) + if err != nil { + log.Errorf("runAppVNC: Error remove file %v", err) + return + } + } + } + log.Functionf("runAppVNC: %v, done", vmiName) +} + +func getVMIdomainName(ctx *zedkubeContext, config *types.AppInstanceConfig) (string, error) { + clientset, err := kubeapi.GetKubevirtClientSet() + if err != nil { + log.Errorf("getVMIs: get virtclient error %v", err) + return "", err + } + + vmiName := base.GetAppKubeName(config.DisplayName, config.UUIDandVersion.UUID) + var domainName string + vmis, err := clientset.VirtualMachineInstance(kubeapi.EVEKubeNameSpace).List(context.Background(), &metav1.ListOptions{}) + if err != nil { + log.Errorf("getVMIs: get VMI list error %v", err) + return "", err + } + + for _, vmi := range vmis.Items { + if !strings.Contains(vmi.ObjectMeta.Name, vmiName) { + continue + } + domainName = vmi.ObjectMeta.Name + break + } + + return domainName, nil +} diff --git a/pkg/pillar/cmd/zedkube/zedkube.go b/pkg/pillar/cmd/zedkube/zedkube.go new file mode 100644 index 00000000000..34915d0b6e7 --- /dev/null +++ b/pkg/pillar/cmd/zedkube/zedkube.go @@ -0,0 +1,200 @@ +// Copyright (c) 2024 Zededa, Inc. +// SPDX-License-Identifier: Apache-2.0 + +//go:build kubevirt + +package zedkube + +import ( + "sync" + "time" + + "github.com/lf-edge/eve/pkg/pillar/agentbase" + "github.com/lf-edge/eve/pkg/pillar/agentlog" + "github.com/lf-edge/eve/pkg/pillar/base" + "github.com/lf-edge/eve/pkg/pillar/kubeapi" + "github.com/lf-edge/eve/pkg/pillar/pubsub" + "github.com/lf-edge/eve/pkg/pillar/types" + "github.com/sirupsen/logrus" + "k8s.io/client-go/rest" +) + +const ( + agentName = "zedkube" + // Time limits for event loop handlers + errorTime = 3 * time.Minute + warningTime = 40 * time.Second + stillRunningInterval = 25 * time.Second + logcollectInterval = 30 + // run VNC file + vmiVNCFileName = "/run/zedkube/vmiVNC.run" +) + +var ( + logger *logrus.Logger + log *base.LogObject +) + +type zedkubeContext struct { + agentbase.AgentBase + globalConfig *types.ConfigItemValueMap + subAppInstanceConfig pubsub.Subscription + subGlobalConfig pubsub.Subscription + networkInstanceStatusMap sync.Map + ioAdapterMap sync.Map + config *rest.Config + appLogStarted bool + appContainerLogger *logrus.Logger +} + +// Run - an zedkube run +func Run(ps *pubsub.PubSub, loggerArg *logrus.Logger, logArg *base.LogObject, arguments []string) int { + logger = loggerArg + log = logArg + + zedkubeCtx := zedkubeContext{ + globalConfig: types.DefaultConfigItemValueMap(), + } + agentbase.Init(&zedkubeCtx, logger, log, agentName, + agentbase.WithPidFile(), + agentbase.WithWatchdog(ps, warningTime, errorTime), + agentbase.WithArguments(arguments)) + + // Run a periodic timer so we always update StillRunning + stillRunning := time.NewTicker(stillRunningInterval) + + zedkubeCtx.appContainerLogger = agentlog.CustomLogInit(logrus.InfoLevel) + + // Get AppInstanceConfig from zedagent + subAppInstanceConfig, err := ps.NewSubscription(pubsub.SubscriptionOptions{ + AgentName: "zedagent", + MyAgentName: agentName, + TopicImpl: types.AppInstanceConfig{}, + Activate: false, + Ctx: &zedkubeCtx, + CreateHandler: handleAppInstanceConfigCreate, + ModifyHandler: handleAppInstanceConfigModify, + DeleteHandler: handleAppInstanceConfigDelete, + WarningTime: warningTime, + ErrorTime: errorTime, + }) + if err != nil { + log.Fatal(err) + } + zedkubeCtx.subAppInstanceConfig = subAppInstanceConfig + subAppInstanceConfig.Activate() + + // Look for global config such as log levels + subGlobalConfig, err := ps.NewSubscription(pubsub.SubscriptionOptions{ + AgentName: "zedagent", + MyAgentName: agentName, + TopicImpl: types.ConfigItemValueMap{}, + Persistent: true, + Activate: false, + Ctx: &zedkubeCtx, + CreateHandler: handleGlobalConfigCreate, + ModifyHandler: handleGlobalConfigModify, + WarningTime: warningTime, + ErrorTime: errorTime, + }) + if err != nil { + log.Fatal(err) + } + zedkubeCtx.subGlobalConfig = subGlobalConfig + subGlobalConfig.Activate() + + err = kubeapi.WaitForKubernetes(agentName, ps, stillRunning) + if err != nil { + log.Errorf("zedkube: WaitForKubenetes %v", err) + } + zedkubeCtx.config, err = kubeapi.GetKubeConfig() + if err != nil { + log.Errorf("zedkube: GetKubeConfi %v", err) + } else { + log.Noticef("zedkube: running") + } + + appLogTimer := time.NewTimer(logcollectInterval * time.Second) + + for { + select { + case change := <-subAppInstanceConfig.MsgChan(): + subAppInstanceConfig.ProcessChange(change) + + case <-appLogTimer.C: + collectAppLogs(&zedkubeCtx) + appLogTimer = time.NewTimer(logcollectInterval * time.Second) + + case change := <-subGlobalConfig.MsgChan(): + subGlobalConfig.ProcessChange(change) + + case <-stillRunning.C: + } + ps.StillRunning(agentName, warningTime, errorTime) + } +} + +func handleAppInstanceConfigCreate(ctxArg interface{}, key string, + configArg interface{}) { + ctx := ctxArg.(*zedkubeContext) + config := configArg.(types.AppInstanceConfig) + + log.Functionf("handleAppInstanceConfigCreate(%v) spec for %s", + config.UUIDandVersion, config.DisplayName) + + err := checkIoAdapterEthernet(ctx, &config) + log.Functionf("handleAppInstancConfigModify: genAISpec %v", err) +} + +func handleAppInstanceConfigModify(ctxArg interface{}, key string, + configArg interface{}, oldConfigArg interface{}) { + ctx := ctxArg.(*zedkubeContext) + config := configArg.(types.AppInstanceConfig) + oldconfig := oldConfigArg.(types.AppInstanceConfig) + + log.Functionf("handleAppInstancConfigModify(%v) spec for %s", + config.UUIDandVersion, config.DisplayName) + + err := checkIoAdapterEthernet(ctx, &config) + + if oldconfig.RemoteConsole != config.RemoteConsole { + log.Functionf("handleAppInstancConfigModify: new remote console %v", config.RemoteConsole) + go runAppVNC(ctx, &config) + } + log.Functionf("handleAppInstancConfigModify: genAISpec %v", err) +} + +func handleAppInstanceConfigDelete(ctxArg interface{}, key string, + configArg interface{}) { + + log.Functionf("handleAppInstanceConfigDelete(%s)", key) + ctx := ctxArg.(*zedkubeContext) + config := configArg.(types.AppInstanceConfig) + + checkDelIoAdpaterEthernet(ctx, &config) + log.Functionf("handleAppInstanceConfigDelete(%s) done", key) +} + +func handleGlobalConfigCreate(ctxArg interface{}, key string, + statusArg interface{}) { + handleGlobalConfigImpl(ctxArg, key, statusArg) +} + +func handleGlobalConfigModify(ctxArg interface{}, key string, + statusArg interface{}, oldStatusArg interface{}) { + handleGlobalConfigImpl(ctxArg, key, statusArg) +} + +func handleGlobalConfigImpl(ctxArg interface{}, key string, + statusArg interface{}) { + + ctx := ctxArg.(*zedkubeContext) + if key != "global" { + log.Functionf("handleGlobalConfigImpl: ignoring %s", key) + return + } + log.Functionf("handleGlobalConfigImpl for %s", key) + _ = agentlog.HandleGlobalConfig(log, ctx.subGlobalConfig, agentName, + ctx.CLIParams().DebugOverride, ctx.Logger()) + log.Functionf("handleGlobalConfigImpl(%s): done", key) +} diff --git a/pkg/pillar/scripts/device-steps.sh b/pkg/pillar/scripts/device-steps.sh index 266e09ff6fd..4b63c6cddbc 100755 --- a/pkg/pillar/scripts/device-steps.sh +++ b/pkg/pillar/scripts/device-steps.sh @@ -18,7 +18,7 @@ ZTMPDIR=/run/global DPCDIR=$ZTMPDIR/DevicePortConfig FIRSTBOOTFILE=$ZTMPDIR/first-boot FIRSTBOOT= -AGENTS="diag zedagent ledmanager nim nodeagent domainmgr loguploader tpmmgr vaultmgr zedmanager zedrouter downloader verifier baseosmgr wstunnelclient volumemgr watcher zfsmanager usbmanager" +AGENTS="diag zedagent ledmanager nim nodeagent domainmgr loguploader tpmmgr vaultmgr zedmanager zedrouter downloader verifier baseosmgr wstunnelclient volumemgr watcher zfsmanager usbmanager zedkube" TPM_DEVICE_PATH="/dev/tpmrm0" PATH=$BINDIR:$PATH TPMINFOTEMPFILE=/var/tmp/tpminfo.txt diff --git a/pkg/pillar/zedbox/zedbox.go b/pkg/pillar/zedbox/zedbox.go index cd74bced519..35d8e65d70c 100644 --- a/pkg/pillar/zedbox/zedbox.go +++ b/pkg/pillar/zedbox/zedbox.go @@ -42,6 +42,7 @@ import ( "github.com/lf-edge/eve/pkg/pillar/cmd/watcher" "github.com/lf-edge/eve/pkg/pillar/cmd/wstunnelclient" "github.com/lf-edge/eve/pkg/pillar/cmd/zedagent" + "github.com/lf-edge/eve/pkg/pillar/cmd/zedkube" "github.com/lf-edge/eve/pkg/pillar/cmd/zedmanager" "github.com/lf-edge/eve/pkg/pillar/cmd/zedrouter" "github.com/lf-edge/eve/pkg/pillar/cmd/zfsmanager" @@ -92,6 +93,7 @@ var ( "volumemgr": {f: volumemgr.Run}, "waitforaddr": {f: waitforaddr.Run, inline: inlineAlways}, "zedagent": {f: zedagent.Run}, + "zedkube": {f: zedkube.Run}, "zedmanager": {f: zedmanager.Run}, "zedrouter": {f: zedrouter.Run}, "ipcmonitor": {f: ipcmonitor.Run, inline: inlineAlways},