Merge pull request #448 from seokho-son/importexport

Expedite auto agent installation

src/core/mcis/control.go

@@ -289,19 +289,30 @@ type TbVmRecommendInfo struct {
 	Placement_param []common.KeyValue `json:"placement_param"`
 }
 
-func VerifySshUserName(nsId string, mcisId string, vmId string, vmIp string, sshPort string, userNames []string, privateKey string) (string, error) {
+func VerifySshUserName(nsId string, mcisId string, vmId string, vmIp string, sshPort string, givenUserName string) (string, string, error) {
 
 	// verify if vm is running with a public ip.
 	if vmIp == "" {
-		return "", fmt.Errorf("Cannot do ssh, VM IP is null")
+		return "", "", fmt.Errorf("Cannot do ssh, VM IP is null")
 	}
 	vmStatusInfoTmp, err := GetVmStatus(nsId, mcisId, vmId)
 	if err != nil {
 		common.CBLog.Error(err)
-		return "", err
+		return "", "", err
 	}
 	if vmStatusInfoTmp.Status != StatusRunning || vmIp == "" {
-		return "", fmt.Errorf("Cannot do ssh, VM IP is not Running")
+		return "", "", fmt.Errorf("Cannot do ssh, VM IP is not Running")
+	}
+
+	// find vaild username
+	userName, _, privateKey := GetVmSshKey(nsId, mcisId, vmId)
+	userNames := []string{
+		userName,
+		givenUserName,
+		SshDefaultUserName01,
+		SshDefaultUserName02,
+		SshDefaultUserName03,
+		SshDefaultUserName04,
 	}
 
 	theUserName := ""
@@ -312,14 +323,28 @@ func VerifySshUserName(nsId string, mcisId string, vmId string, vmIp string, ssh
 	if verifiedUserName != "" {
 		fmt.Println("[SSH] " + "(" + vmIp + ")" + "with userName:" + verifiedUserName)
 		fmt.Println("[CMD] " + cmd)
+
+		retrycheck := 10
+		for i := 0; i < retrycheck; i++ {
+			conerr := CheckConnectivity(vmIp, sshPort)
+			if conerr == nil {
+				fmt.Println("[ERR: conerr] nil. break")
+				break
+			}
+			if i == retrycheck-1 {
+				return "", "", fmt.Errorf("Cannot do ssh, the port is not opened (10 trials)")
+			}
+			time.Sleep(11 * time.Second)
+		}
+
 		result, err := RunSSH(vmIp, sshPort, verifiedUserName, privateKey, cmd)
 		if err != nil {
 			fmt.Println("[ERR: result] " + "[ERR: err] " + err.Error())
 		}
 		if err == nil {
 			theUserName = verifiedUserName
 			fmt.Println("[RST] " + *result + "[Username] " + verifiedUserName)
-			return theUserName, nil
+			return theUserName, privateKey, nil
 		}
 	}
 
@@ -351,7 +376,7 @@ func VerifySshUserName(nsId string, mcisId string, vmId string, vmIp string, ssh
 		time.Sleep(1 * time.Second)
 	}
 
-	return theUserName, nil
+	return theUserName, privateKey, nil
 }
 
 type SshCmdResult struct { // Tumblebug
@@ -418,17 +443,7 @@ func InstallAgentToMcis(nsId string, mcisId string, req *McisCmdReq) (AgentInsta
 		// }
 
 		// find vaild username
-		userName, _, sshKey := GetVmSshKey(nsId, mcisId, vmId)
-		userNames := []string{
-			userName,
-			req.User_name,
-			SshDefaultUserName01,
-			SshDefaultUserName02,
-			SshDefaultUserName03,
-			SshDefaultUserName04,
-		}
-
-		userName, err = VerifySshUserName(nsId, mcisId, vmId, vmIp, sshPort, userNames, sshKey)
+		userName, sshKey, err := VerifySshUserName(nsId, mcisId, vmId, vmIp, sshPort, req.User_name)
 
 		fmt.Println("[SSH] " + mcisId + "/" + vmId + "(" + vmIp + ")" + "with userName:" + userName)
 		fmt.Println("[CMD] " + cmd)
@@ -1669,16 +1684,8 @@ func CorePostCmdMcisVm(nsId string, mcisId string, vmId string, req *McisCmdReq)
 	cmd := req.Command
 
 	// find vaild username
-	userName, _, sshKey := GetVmSshKey(nsId, mcisId, vmId)
-	userNames := []string{
-		userName,
-		req.User_name,
-		SshDefaultUserName01,
-		SshDefaultUserName02,
-		SshDefaultUserName03,
-		SshDefaultUserName04,
-	}
-	userName, err := VerifySshUserName(nsId, mcisId, vmId, vmIp, sshPort, userNames, sshKey)
+	userName, sshKey, err := VerifySshUserName(nsId, mcisId, vmId, vmIp, sshPort, req.User_name)
+
 	if userName == "" {
 		//return c.JSON(http.StatusInternalServerError, errors.New("No vaild username"))
 		return "", fmt.Errorf("No vaild username")
@@ -1756,16 +1763,7 @@ func CorePostCmdMcis(nsId string, mcisId string, req *McisCmdReq) ([]SshCmdResul
 		// 	userName = sshDefaultUserName
 		// }
 		// find vaild username
-		userName, _, sshKey := GetVmSshKey(nsId, mcisId, vmId)
-		userNames := []string{
-			userName,
-			req.User_name,
-			SshDefaultUserName01,
-			SshDefaultUserName02,
-			SshDefaultUserName03,
-			SshDefaultUserName04,
-		}
-		userName, err = VerifySshUserName(nsId, mcisId, vmId, vmIp, sshPort, userNames, sshKey)
+		userName, sshKey, err := VerifySshUserName(nsId, mcisId, vmId, vmIp, sshPort, req.User_name)
 
 		fmt.Println("[SSH] " + mcisId + "/" + vmId + "(" + vmIp + ")" + "with userName:" + userName)
 		fmt.Println("[CMD] " + cmd)
@@ -2303,6 +2301,7 @@ func CreateMcis(nsId string, req *TbMcisReq) string {
 			reqToMon := &McisCmdReq{}
 			reqToMon.User_name = "ubuntu" // this MCIS user name is temporal code. Need to improve.
 
+			fmt.Printf("\n===========================\n")
 			fmt.Printf("\n[InstallMonitorAgentToMcis]\n\n")
 			content, err := InstallMonitorAgentToMcis(nsId, mcisId, reqToMon)
 			if err != nil {

src/core/mcis/monitor.go

@@ -98,10 +98,24 @@ func CheckDragonflyEndpoint() error {
 	return nil
 }
 
-func CallMonitoringAsync(wg *sync.WaitGroup, nsID string, mcisID string, vmID string, vmIP string, sshPort string, userName string, privateKey string, method string, cmd string, returnResult *[]SshCmdResult) {
+func CallMonitoringAsync(wg *sync.WaitGroup, nsID string, mcisID string, vmID string, givenUserName string, method string, cmd string, returnResult *[]SshCmdResult) {
 
 	defer wg.Done() //goroutin sync done
 
+	vmIP, sshPort := GetVmIp(nsID, mcisID, vmID)
+	userName, privateKey, err := VerifySshUserName(nsID, mcisID, vmID, vmIP, sshPort, givenUserName)
+	errStr := ""
+	if err != nil {
+		common.CBLog.Error(err)
+		errStr = err.Error()
+	}
+	fmt.Println("[CallMonitoringAsync] " + mcisID + "/" + vmID + "(" + vmIP + ")" + "with userName:" + userName)
+
+	// set vm MonAgentStatus = "installing" (to avoid duplicated requests)
+	vmInfoTmp, _ := GetVmObject(nsID, mcisID, vmID)
+	vmInfoTmp.MonAgentStatus = "installing"
+	UpdateVmInfo(nsID, mcisID, vmInfoTmp)
+
 	url := common.DRAGONFLY_REST_URL + cmd
 	fmt.Println("\n[Calling DRAGONFLY] START")
 	fmt.Println("url: " + url + " method: " + method)
@@ -115,8 +129,10 @@ func CallMonitoringAsync(wg *sync.WaitGroup, nsID string, mcisID string, vmID st
 		User_name: userName,
 		Ssh_key:   privateKey,
 	}
-	fmt.Printf("\n[Request body to CB-DRAGONFLY]\n")
-	common.PrintJsonPretty(tempReq)
+	if tempReq.Ssh_key == "" {
+		fmt.Printf("\n[Request body to CB-DRAGONFLY]A problem detected.Ssh_key is empty.\n")
+		common.PrintJsonPretty(tempReq)
+	}
 
 	payload := &bytes.Buffer{}
 	writer := multipart.NewWriter(payload)
@@ -128,9 +144,8 @@ func CallMonitoringAsync(wg *sync.WaitGroup, nsID string, mcisID string, vmID st
 	_ = writer.WriteField("user_name", userName)
 	_ = writer.WriteField("ssh_key", privateKey)
 	_ = writer.WriteField("cspType", "test")
-	err := writer.Close()
+	err = writer.Close()
 
-	errStr := ""
 	if err != nil {
 		common.CBLog.Error(err)
 		errStr = err.Error()
@@ -173,7 +188,7 @@ func CallMonitoringAsync(wg *sync.WaitGroup, nsID string, mcisID string, vmID st
 
 	//wg.Done() //goroutin sync done
 
-	vmInfoTmp, _ := GetVmObject(nsID, mcisID, vmID)
+	//vmInfoTmp, _ := GetVmObject(nsID, mcisID, vmID)
 
 	sshResultTmp := SshCmdResult{}
 	sshResultTmp.Mcis_id = mcisID
@@ -235,29 +250,13 @@ func InstallMonitorAgentToMcis(nsId string, mcisId string, req *McisCmdReq) (Age
 		vmObjTmp, _ := GetVmObject(nsId, mcisId, v)
 		fmt.Println("MonAgentStatus : " + vmObjTmp.MonAgentStatus)
 
-		if vmObjTmp.MonAgentStatus != "installed" {
-
-			vmId := v
-			vmIp, sshPort := GetVmIp(nsId, mcisId, vmId)
-
-			// find vaild username
-			userName, _, sshKey := GetVmSshKey(nsId, mcisId, vmId)
-			userNames := []string{
-				userName,
-				req.User_name,
-				SshDefaultUserName01,
-				SshDefaultUserName02,
-				SshDefaultUserName03,
-				SshDefaultUserName04,
-			}
-			userName, err = VerifySshUserName(nsId, mcisId, vmId, vmIp, sshPort, userNames, sshKey)
-
-			fmt.Println("[CallMonitoringAsync] " + mcisId + "/" + vmId + "(" + vmIp + ")" + "with userName:" + userName)
+		// Request agent installation (skip if in installing or installed status)
+		if vmObjTmp.MonAgentStatus != "installed" && vmObjTmp.MonAgentStatus != "installing" {
 
 			// Avoid RunSSH to not ready VM
 			if err == nil {
 				wg.Add(1)
-				go CallMonitoringAsync(&wg, nsId, mcisId, vmId, vmIp, sshPort, userName, sshKey, method, cmd, &resultArray)
+				go CallMonitoringAsync(&wg, nsId, mcisId, v, req.User_name, method, cmd, &resultArray)
 			} else {
 				common.CBLog.Error(err)
 			}

src/core/mcis/sshrun.go

@@ -13,9 +13,12 @@ import (
 
 	//"github.com/sirupsen/logrus"
 
+	"fmt"
 	"io"
+	"net"
 	"os"
 	"strings"
+	"time"
 
 	"github.com/bramvdbogaerde/go-scp"
 	"github.com/bramvdbogaerde/go-scp/auth"
@@ -180,3 +183,22 @@ func SSHCopyByKeyPath(sshInfo SSHKeyPathInfo, sourcePath string, remotePath stri
 
 	return Copy(sshCli, sourcePath, remotePath)
 }
+
+// CheckConnectivity func checks if given port is open and ready.
+// For instance, ready for ssh port can be checkek.
+func CheckConnectivity(host string, port string) error {
+
+	deadline := 10
+	timeout := time.Second * time.Duration(deadline)
+	conn, err := net.DialTimeout("tcp", net.JoinHostPort(host, port), timeout)
+	if err != nil {
+		fmt.Println("[CheckConnectivity]", host, ":", port, ". ERR:", err)
+		return err
+	}
+	if conn != nil {
+		defer conn.Close()
+		fmt.Println("[CheckConnectivity]", host, ":", port, ". Opened")
+		return nil
+	}
+	return nil
+}

src/main.go

@@ -64,7 +64,9 @@ func main() {
 	ticker := time.NewTicker(time.Millisecond * time.Duration(autoControlDuration))
 	go func() {
 		for t := range ticker.C {
-			fmt.Println("- Orchestration Controller ", t.Format("2006-01-02 15:04:05"))
+			//display ticker if you need (remove '_ = t')
+			_ = t
+			//fmt.Println("- Orchestration Controller ", t.Format("2006-01-02 15:04:05"))
 			mcis.OrchestrationController()
 		}
 	}()

test/official/9.monitoring/get-monitoring-data.sh

@@ -22,6 +22,11 @@
 	getCloudIndex $CSP
 
 	MCISID=${CONN_CONFIG[$INDEX,$REGION]}-${POSTFIX}
+	if [ "${INDEX}" == "0" ]; then
+		MCISPREFIX=avengers
+		MCISID=${MCISPREFIX}-${POSTFIX}
+	fi
+
 	USERCMD=${4}
 
 	curl -H "${AUTH}" -sX GET http://$TumblebugServer/tumblebug/ns/$NS_ID/monitoring/mcis/$MCISID/metric/$USERCMD | json_pp #|| return 1

test/official/9.monitoring/install-agent.sh

@@ -22,6 +22,12 @@
 	getCloudIndex $CSP
 
 	MCISID=${CONN_CONFIG[$INDEX,$REGION]}-${POSTFIX}
+
+	if [ "${INDEX}" == "0" ]; then
+		MCISPREFIX=avengers
+		MCISID=${MCISPREFIX}-${POSTFIX}
+	fi
+
 	curl -H "${AUTH}" -sX POST http://$TumblebugServer/tumblebug/ns/$NS_ID/monitoring/install/mcis/$MCISID -H 'Content-Type: application/json' -d \
 		'{
 			"command": "echo -n [CMD] Works! [Public IP: ; curl https://api.ipify.org ; echo -n ], [Hostname: ; hostname ; echo -n ]"