Add kubernetes remote debugging capability

Signed-off-by: Naiming Shen <naiming@Admins-MacBook-Pro-3.local>

pkg/edgeview/go.mod

@@ -13,6 +13,7 @@ require (
 	github.com/sirupsen/logrus v1.9.0
 	github.com/tatsushid/go-fastping v0.0.0-20160109021039-d7bb493dee3e
 	github.com/vishvananda/netlink v1.1.1-0.20210924202909-187053b97868
+	golang.org/x/crypto v0.14.0
 	golang.org/x/sys v0.18.0
 	golang.org/x/time v0.3.0
 )

pkg/edgeview/go.sum

@@ -822,6 +822,8 @@ golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
 golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc=
+golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -1049,6 +1051,7 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
 golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

pkg/edgeview/src/basics.go

@@ -143,6 +143,53 @@ func initOpts() {
 	}
 }
 
+// this script is automatically installed into your /tmp/download/bin
+// and can be used to run your kubectl with the download kubeconfig file
+// decrypted by the ssh private key
+const kubeConfdecrpytScript = `#!/bin/bash
+
+usage() {
+    echo "Usage: $0 [-keypath=\"your ssh private key file path\"]"
+}
+
+# Parse command line arguments
+while [[ $# -gt 0 ]]; do
+    key="$1"
+
+    case $key in
+        -keypath=*)
+            keypath="${key#*=}"
+            shift # Shift to the next argument after the keypath value
+            ;;
+        *)
+            # Unknown option or argument
+            usage
+            exit 1
+            ;;
+    esac
+done
+
+# Set the private key path based on the provided -keypath or use the default
+if [ -n "$keypath" ]; then
+    privateKeyFile="$keypath"
+else
+    # use default ssh private key
+    privateKeyFile="$HOME/.ssh/id_rsa"
+fi
+
+# the symmetric key is encrypted by ssh public key
+symmetricKeyEncFile="/tmp/download/kube-symmetric-file.enc"
+# the kubeconfig file is encrypted by the symmetric key
+symmetricEncFile="/tmp/download/kube-config-yaml"
+
+# Decrypt the symmetric key using the SSH private key
+symmetricKey=$(openssl pkeyutl -decrypt -inkey "$privateKeyFile" -in "$symmetricKeyEncFile")
+
+# decrypt the kube config with openssl
+kconfig=$(openssl enc -aes-256-cbc -d -in "$symmetricEncFile" -k "$symmetricKey" 2>/dev/null)
+
+echo "$kconfig"`
+
 // checkOpts -
 // a pre-defined sets of 'network', 'system', 'pub' commands are supported, the command options can be
 // multiple and separated by ',', this function to verify each of the command is valid and supported
@@ -585,6 +632,24 @@ func listRecursiveFiles(path, pattern string) ([]string, error) {
 	return jfiles, nil
 }
 
+// this script is used to run your kubectl with the download kubeconfig file
+// decrypted by the ssh private key
+func checkAndInstallKubeDecryptScript() error {
+	scriptfile := filepath.Join(fileCopyDir, "bin", "edgeview-kube-decrypt.sh")
+	err := os.MkdirAll(filepath.Dir(scriptfile), os.ModePerm)
+	if err != nil {
+		fmt.Println("Error creating directory:", err)
+		return err
+	}
+
+	if err := os.WriteFile(scriptfile, []byte(kubeConfdecrpytScript), 0755); err != nil {
+		fmt.Println("Error writing script to file:", err)
+		return err
+	}
+
+	return nil
+}
+
 var helpStr = `eve-edgeview [ -token <session-token> ] [ -inst <instance-id> ] <query command>
  query options:
 `

pkg/edgeview/src/copyfile.go

@@ -7,6 +7,7 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"io"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -36,6 +37,7 @@ const (
 	copyLogFiles
 	copyTarFiles
 	copyTechSupport
+	copyKubeConfig
 )
 
 var (
@@ -381,6 +383,8 @@ func recvCopyFile(msg []byte, fstatus *fileCopyStatus, mtype int) {
 			} else {
 				fmt.Printf("\nfile size %d, saved at %s\n", fstatus.currSize, fileCopyDir+fileNameClean)
 			}
+		} else if fstatus.cType == copyKubeConfig {
+			splitKubeConfigFiles(fstatus.filename)
 		}
 		transferStr := fmt.Sprintf("\n file %s size %d", fileNameClean, fstatus.currSize)
 		if serverSentSize != 0 && fstatus.currSize != int64(serverSentSize) {
@@ -442,6 +446,77 @@ func sendCopyDone(context string, err error) {
 	}
 }
 
+// split files into key and encrypted files
+// since the edgeview copy operation only downloaded a combined file
+// you need your ssh private key to descript the symmetric key file, and use that
+// to decrypt the kubeconfig file
+func splitKubeConfigFiles(combFile string) {
+	fileStrs := strings.Split(combFile, ".")
+	if len(fileStrs) != 2 {
+		fmt.Printf("get file name incorrect %s\n", combFile)
+		return
+	}
+	numBytes := fileStrs[1]
+
+	bytesPlusOne, err := strconv.Atoi(numBytes)
+	if err != nil {
+		fmt.Printf("get file name incorrect num %s\n", numBytes)
+		return
+	}
+
+	// Open the combined file
+	combFilePath := filepath.Join(fileCopyDir, combFile)
+	cleanCombFilePath := filepath.Clean(combFilePath)
+	// To fix CodeQL warning, Check if the cleaned path is still within the intended directory
+	if !strings.HasPrefix(cleanCombFilePath, fileCopyDir) {
+		fmt.Println("potential path traversal attempt detected")
+		return
+	}
+
+	combFileHandle, err := os.Open(cleanCombFilePath)
+	if err != nil {
+		fmt.Printf("error opening combined file: %v\n", err)
+		return
+	}
+	defer combFileHandle.Close()
+
+	// Create the symKeyClientFile
+	symFileHandle, err := os.Create(symKeyClientFile)
+	if err != nil {
+		fmt.Printf("error creating sym file: %v\n", err)
+		return
+	}
+	defer symFileHandle.Close()
+
+	// Copy the first 'bytesPlusOne' bytes from the combined file to the symKeyClientFile
+	_, err = io.CopyN(symFileHandle, combFileHandle, int64(bytesPlusOne))
+	if err != nil {
+		fmt.Printf("error copying to sym file: %v\n", err)
+		return
+	}
+
+	// Create the kubeClientFile
+	kubeFileHandle, err := os.Create(kubeClientFile)
+	if err != nil {
+		fmt.Printf("error creating kube file: %v\n", err)
+		return
+	}
+	defer kubeFileHandle.Close()
+
+	// Copy the rest of the combined file to the kubeClientFile
+	_, err = io.Copy(kubeFileHandle, combFileHandle)
+	if err != nil {
+		fmt.Printf("error copying to kube file: %v\n", err)
+		return
+	}
+
+	// Remove the combined file
+	err = os.Remove(cleanCombFilePath)
+	if err != nil {
+		fmt.Printf("error removing combined file: %v\n", err)
+	}
+}
+
 // untarLogfile - unzip and make into a single .txt
 // with sequential log entries for dev and each of the apps
 // this is done only if the tar file size is not too large

pkg/edgeview/src/edge-view.go

@@ -37,6 +37,7 @@ var (
 	tcpRl            *rate.Limiter
 	disableRateLimit bool // ratelimit can be disabled by the message from dispatcher
 	lostClientPeer   bool // dispatcher will send 'no device online' if peer lost
+	IsHVTypeKube     bool // IsHVTypeKube hypervisor type is kubernetes
 )
 
 const (
@@ -117,6 +118,7 @@ func main() {
 	var fstatus fileCopyStatus
 	remotePorts := make(map[int]int)
 	var tcpclientCnt int
+	var kubecfg bool
 	var pqueryopt, pnetopt, psysopt, ppubsubopt, logopt, timeopt string
 	var jsonopt bool
 	typeopt := "all"
@@ -214,10 +216,17 @@ func main() {
 			psysopt = pqueryopt
 		} else if strings.HasPrefix(pqueryopt, "tcp/") {
 			var ok bool
-			ok, tcpclientCnt, remotePorts = processTCPcmd(pqueryopt, remotePorts)
+			ok, kubecfg, tcpclientCnt, remotePorts = processTCPcmd(pqueryopt, remotePorts)
 			if !ok {
 				return
 			}
+			if kubecfg {
+				fstatus.cType = copyKubeConfig
+				err := checkAndInstallKubeDecryptScript()
+				if err != nil {
+					return
+				}
+			}
 			pnetopt = pqueryopt
 		} else if strings.HasPrefix(pqueryopt, "cp/") {
 			psysopt = pqueryopt
@@ -301,6 +310,7 @@ func main() {
 	// 2) websocket client mode, runs on operator/laptop side
 	if runOnServer { // 1) websocket mode on device 'server' side
 
+		IsHVTypeKube = base.IsHVTypeKube()
 		err := initPolicy()
 		if err != nil {
 			log.Noticef("edgeview exit, init policy err. %v", err)
@@ -501,9 +511,11 @@ func main() {
 			waitPulish = false
 			doInfoPub(infoPub)
 		case <-done:
+			delKubeConfigFile(kubecfg)
 			tcpClientSendDone()
 			return
 		case <-intSignal:
+			delKubeConfigFile(kubecfg)
 			tcpClientSendDone()
 			return
 		}

pkg/edgeview/src/policy.go

@@ -7,6 +7,7 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"net"
 	"os"
 	"strconv"
 	"strings"
@@ -18,6 +19,7 @@ const (
 	devPolicyErr = "EVE policy not allow"
 	appPolicyErr = "App policy not allow"
 	extPolicyErr = "External policy not allow"
+	kubPolicyErr = "Kubernetes policy not allow"
 	vncPolicyErr = "App VNC access must be enabled"
 	tcpSyntaxErr = "TCP syntax error"
 )
@@ -28,6 +30,7 @@ var (
 	devPolicy  types.EvDevPolicy
 	appPolicy  types.EvAppPolicy
 	extPolicy  types.EvExtPolicy
+	//kubPolicy  types.EvKubPolicy // XXX may enable this in the future
 )
 
 func initPolicy() error {
@@ -180,7 +183,7 @@ func checkTCPPolicy(tcpOpts string, evStatus *types.EdgeviewStatus) (bool, strin
 // One TCP cmd with multiple address:port, count for multiple access
 // E.g. tcp/proxy/localhost:22/10.1.0.102:5901 count access for device 1, and app 2
 func checkIPportPolicy(tcpOpt string, evStatus *types.EdgeviewStatus) (bool, string, string) {
-	if strings.HasPrefix(tcpOpt, "proxy") {
+	if strings.HasPrefix(tcpOpt, "proxy") || strings.HasPrefix(tcpOpt, "kube") {
 		// 'proxy' sessions will be check at connect time
 		return true, "", ""
 	}
@@ -192,6 +195,7 @@ func checkIPportPolicy(tcpOpt string, evStatus *types.EdgeviewStatus) (bool, str
 		}
 		ipaddr := opts[0]
 		ipport := opts[1]
+		isAddrKube := checkAddrKube(ipaddr)
 		isAddrDevice := checkAddrLocal(ipaddr)
 		// check console access for apps first
 		isAppConsole, allowVNC, name := checkAppConsole(ipaddr, ipport)
@@ -209,6 +213,11 @@ func checkIPportPolicy(tcpOpt string, evStatus *types.EdgeviewStatus) (bool, str
 			} else {
 				evStatus.CmdCountDev++
 			}
+		} else if isAddrKube {
+			// XXX can check on this policy in the future if needed
+			//if !kubPolicy.Enabled {
+			//	return false, "", kubPolicyErr
+			//}
 		} else { // App Interface IP
 			isAddrApps, vncEnable, name := checkAddrApps(ipaddr)
 			if isAddrApps {
@@ -238,6 +247,27 @@ func checkIPportPolicy(tcpOpt string, evStatus *types.EdgeviewStatus) (bool, str
 	return true, appName, ""
 }
 
+// checkAddrKube - check if the IP address is in the kube network
+// those are the k3s specific default network prefixes
+func checkAddrKube(addr string) bool {
+	ipa := net.ParseIP(addr)
+	if ipa == nil {
+		return false
+	}
+
+	_, subnet1, err := net.ParseCIDR("10.42.0.0/16")
+	if err != nil {
+		return false
+	}
+
+	_, subnet2, err := net.ParseCIDR("10.43.0.0/16")
+	if err != nil {
+		return false
+	}
+
+	return subnet1.Contains(ipa) || subnet2.Contains(ipa)
+}
+
 func checkAddrLocal(addr string) bool {
 	for _, a := range devIntfIPs {
 		if a == addr {

pkg/edgeview/src/system.go

@@ -36,6 +36,7 @@ var tarBlockDirs = []string{
 	"/persist/clear",
 	"/persist/vault",
 	"/run/domainmgr/cloudinit",
+	"/run/.kube/k3s",
 }
 
 var tarBlockFileSuffix = []string{