diff --git a/scripts/runMapUI.sh b/scripts/runMapUI.sh index b13791f28..be091a515 100755 --- a/scripts/runMapUI.sh +++ b/scripts/runMapUI.sh @@ -1,7 +1,7 @@ #!/bin/bash CONTAINER_NAME_READ="CB-MapUI" -CONTAINER_VERSION="0.5.1" +CONTAINER_VERSION="0.5.2" CONTAINER_PORT="-p 1324:1324" CONTAINER_DATA_PATH="" diff --git a/src/api/rest/server/common/utility.go b/src/api/rest/server/common/utility.go index 9d9b97442..3caf1cd22 100644 --- a/src/api/rest/server/common/utility.go +++ b/src/api/rest/server/common/utility.go @@ -376,7 +376,7 @@ func RestInspectResources(c echo.Context) error { type RestRegisterCspNativeResourcesRequest struct { ConnectionName string `json:"connectionName" example:"aws-ap-southeast-1"` NsId string `json:"nsId" example:"ns01"` - McisName string `json:"mcisName" example:"mcis-csp-native"` + McisName string `json:"mcisName" example:"csp"` } // RestRegisterCspNativeResources godoc @@ -385,7 +385,8 @@ type RestRegisterCspNativeResourcesRequest struct { // @Tags [Admin] System management // @Accept json // @Produce json -// @Param Request body RestRegisterCspNativeResourcesRequest true "Specify connectionName and NS Id" +// @Param Request body RestRegisterCspNativeResourcesRequest true "Specify connectionName, NS Id, and MCIS Name"" +// @Param option query string false "Option to specify resourceType" Enums(onlyVm, exceptVm) // @Success 200 {object} mcis.RegisterResourceResult // @Failure 404 {object} common.SimpleMsg // @Failure 500 {object} common.SimpleMsg @@ -396,8 +397,9 @@ func RestRegisterCspNativeResources(c echo.Context) error { if err := c.Bind(u); err != nil { return err } + option := c.QueryParam("option") - content, err := mcis.RegisterCspNativeResources(u.NsId, u.ConnectionName, u.McisName) + content, err := mcis.RegisterCspNativeResources(u.NsId, u.ConnectionName, u.McisName, option) if err != nil { common.CBLog.Error(err) @@ -409,13 +411,20 @@ func RestRegisterCspNativeResources(c echo.Context) error { } +// Request struct for RestRegisterCspNativeResources +type RestRegisterCspNativeResourcesRequestAll struct { + NsId string `json:"nsId" example:"ns01"` + McisName string `json:"mcisName" example:"csp"` +} + // RestRegisterCspNativeResourcesAll godoc // @Summary Register CSP Native Resources (vNet, securityGroup, sshKey, vm) from all Clouds to CB-Tumblebug // @Description Register CSP Native Resources (vNet, securityGroup, sshKey, vm) from all Clouds to CB-Tumblebug // @Tags [Admin] System management // @Accept json // @Produce json -// @Param Request body RestRegisterCspNativeResourcesRequest true "Specify NS Id and MCIS Name" +// @Param Request body RestRegisterCspNativeResourcesRequestAll true "Specify NS Id and MCIS Name" +// @Param option query string false "Option to specify resourceType" Enums(onlyVm, exceptVm) // @Success 200 {object} mcis.RegisterResourceAllResult // @Failure 404 {object} common.SimpleMsg // @Failure 500 {object} common.SimpleMsg @@ -426,8 +435,9 @@ func RestRegisterCspNativeResourcesAll(c echo.Context) error { if err := c.Bind(u); err != nil { return err } + option := c.QueryParam("option") - content, err := mcis.RegisterCspNativeResourcesAll(u.NsId, u.McisName) + content, err := mcis.RegisterCspNativeResourcesAll(u.NsId, u.McisName, option) if err != nil { common.CBLog.Error(err) diff --git a/src/core/common/utility.go b/src/core/common/utility.go index fea9a752c..c3214572f 100644 --- a/src/core/common/utility.go +++ b/src/core/common/utility.go @@ -50,8 +50,14 @@ func GenUid() string { return uid.New().String() } -// RandomSleep is func to make a caller waits for during random time seconds -func RandomSleep(t int) { +// RandomSleep is func to make a caller waits for during random time seconds (random value within x~y) +func RandomSleep(from int, to int) { + if from > to { + tmp := from + from = to + to = tmp + } + t := to - from rand.Seed(time.Now().UnixNano()) n := rand.Intn(t * 1000) time.Sleep(time.Duration(n) * time.Millisecond) diff --git a/src/core/mcir/common.go b/src/core/mcir/common.go index e6cd1af21..9e38bb3f2 100644 --- a/src/core/mcir/common.go +++ b/src/core/mcir/common.go @@ -779,7 +779,7 @@ func ListResource(nsId string, resourceType string) (interface{}, error) { return nil, err } - fmt.Println("[Get " + resourceType + " list") + fmt.Println("[Get] " + resourceType + " list") key := "/ns/" + nsId + "/resources/" + resourceType fmt.Println(key) @@ -1365,7 +1365,7 @@ func LoadCommonResource() (common.IdList, error) { go func(i int, row []string) { defer wait.Done() // RandomSleep for safe parallel executions - common.RandomSleep(20) + common.RandomSleep(0, 20) specReqTmp := TbSpecReq{} // [0]connectionName, [1]cspSpecName, [2]CostPerHour, [3]evaluationScore01, ..., [12]evaluationScore10 specReqTmp.ConnectionName = row[2] @@ -1470,7 +1470,7 @@ func LoadCommonResource() (common.IdList, error) { go func(i int, row []string) { defer wait.Done() // RandomSleep for safe parallel executions - common.RandomSleep(20) + common.RandomSleep(0, 20) imageReqTmp := TbImageReq{} // row0: ProviderName // row1: connectionName diff --git a/src/core/mcir/securitygroup.go b/src/core/mcir/securitygroup.go index fb011d976..8d60037a0 100644 --- a/src/core/mcir/securitygroup.go +++ b/src/core/mcir/securitygroup.go @@ -235,19 +235,21 @@ func CreateSecurityGroup(nsId string, u *TbSecurityGroupReq, option string) (TbS tempReq.ReqInfo.CSPId = u.CspSecurityGroupId // tempReq.ReqInfo.SecurityRules = u.FirewallRules - for _, v := range *u.FirewallRules { - jsonBody, err := json.Marshal(v) - if err != nil { - common.CBLog.Error(err) - } + if u.FirewallRules != nil { + for _, v := range *u.FirewallRules { + jsonBody, err := json.Marshal(v) + if err != nil { + common.CBLog.Error(err) + } - spiderSecurityRuleInfo := SpiderSecurityRuleInfo{} - err = json.Unmarshal(jsonBody, &spiderSecurityRuleInfo) - if err != nil { - common.CBLog.Error(err) - } + spiderSecurityRuleInfo := SpiderSecurityRuleInfo{} + err = json.Unmarshal(jsonBody, &spiderSecurityRuleInfo) + if err != nil { + common.CBLog.Error(err) + } - tempReq.ReqInfo.SecurityRules = append(tempReq.ReqInfo.SecurityRules, spiderSecurityRuleInfo) + tempReq.ReqInfo.SecurityRules = append(tempReq.ReqInfo.SecurityRules, spiderSecurityRuleInfo) + } } var tempSpiderSecurityInfo *SpiderSecurityInfo diff --git a/src/core/mcis/provisioning.go b/src/core/mcis/provisioning.go index 2544ecc19..f53b97ac4 100644 --- a/src/core/mcis/provisioning.go +++ b/src/core/mcis/provisioning.go @@ -1418,7 +1418,7 @@ func CreateVm(nsId string, mcisId string, vmInfoData *TbVmInfo, option string) e payload, _ := json.Marshal(tempReq) // Randomly sleep within 30 Secs to avoid rateLimit from CSP - common.RandomSleep(30) + common.RandomSleep(0, 30) // Call cb-spider API by REST or gRPC if os.Getenv("SPIDER_CALL_METHOD") == "REST" { diff --git a/src/core/mcis/utility.go b/src/core/mcis/utility.go index 23fe78674..42551a5c1 100644 --- a/src/core/mcis/utility.go +++ b/src/core/mcis/utility.go @@ -18,6 +18,7 @@ import ( //"encoding/json" //uuid "github.com/google/uuid" "fmt" + "sort" "strconv" "strings" @@ -563,6 +564,7 @@ func InspectResources(connConfig string, resourceType string) (InspectResource, // RegisterCspNativeResourceResultAll is struct for Register Csp Native Resource Result for All Clouds type RegisterResourceAllResult struct { + ElapsedTime int `json:"elapsedTime"` RegisterationResult []RegisterResourceResult `json:"registerationResult"` } @@ -584,7 +586,8 @@ type registerationOverview struct { } // RegisterCspNativeResourcesAll func registers all CSP-native resources into CB-TB -func RegisterCspNativeResourcesAll(nsId string, mcisId string) (RegisterResourceAllResult, error) { +func RegisterCspNativeResourcesAll(nsId string, mcisId string, option string) (RegisterResourceAllResult, error) { + startTime := time.Now() connectionConfigList, err := common.GetConnConfigList() if err != nil { @@ -592,24 +595,31 @@ func RegisterCspNativeResourcesAll(nsId string, mcisId string) (RegisterResource common.CBLog.Error(err) return RegisterResourceAllResult{}, err } - refinedConnectionConfigList := common.ConnConfigList{} - for _, k := range connectionConfigList.Connectionconfig { - if !strings.Contains(k.ConfigName, "gcp") { - refinedConnectionConfigList.Connectionconfig = append(refinedConnectionConfigList.Connectionconfig, k) - } - } output := RegisterResourceAllResult{} var wait sync.WaitGroup - for _, k := range refinedConnectionConfigList.Connectionconfig { + for _, k := range connectionConfigList.Connectionconfig { wait.Add(1) go func(k common.ConnConfig) { defer wait.Done() mcisNameForRegister := mcisId + "-" + k.ConfigName - //common.RandomSleep(300) - registerResult, err := RegisterCspNativeResources(nsId, k.ConfigName, mcisNameForRegister) + // Assign RandomSleep range by clouds + // This code is temporal, CB-Spider needs to be enhnaced for locking mechanism. + if option != "onlyVm" { + if strings.Contains(k.ConfigName, "alibaba") { + common.RandomSleep(100, 200) + } else if strings.Contains(k.ConfigName, "aws") { + common.RandomSleep(300, 500) + } else if strings.Contains(k.ConfigName, "gcp") { + common.RandomSleep(700, 900) + } else { + } + } + common.RandomSleep(0, 50) + + registerResult, err := RegisterCspNativeResources(nsId, k.ConfigName, mcisNameForRegister, option) if err != nil { common.CBLog.Error(err) } @@ -620,11 +630,17 @@ func RegisterCspNativeResourcesAll(nsId string, mcisId string) (RegisterResource } wait.Wait() + output.ElapsedTime = int(math.Round(time.Now().Sub(startTime).Seconds())) + + sort.SliceStable(output.RegisterationResult, func(i, j int) bool { + return output.RegisterationResult[i].ConnectionName < output.RegisterationResult[j].ConnectionName + }) + return output, err } // RegisterCspNativeResources func registers all CSP-native resources into CB-TB -func RegisterCspNativeResources(nsId string, connConfig string, mcisId string) (RegisterResourceResult, error) { +func RegisterCspNativeResources(nsId string, connConfig string, mcisId string, option string) (RegisterResourceResult, error) { startTime := time.Now() optionFlag := "register" @@ -632,145 +648,153 @@ func RegisterCspNativeResources(nsId string, connConfig string, mcisId string) ( result := RegisterResourceResult{} startTime01 := time.Now() //tmp + var err error - // bring vNet list and register all - inspectedResources, err := InspectResources(connConfig, common.StrVNet) - if err != nil { - common.CBLog.Error(err) - result.SystemMessage = err.Error() - } - for _, r := range inspectedResources.Resources.OnCspOnly.Info { - req := mcir.TbVNetReq{} - req.ConnectionName = connConfig - req.CspVNetId = r.IdByCsp - req.Description = "Ref name: " + r.RefNameOrId + ". CSP managed resource (registered to CB-TB)" - req.Name = req.ConnectionName + "-" + req.CspVNetId - req.Name = strings.ToLower(req.Name) - - _, err = mcir.CreateVNet(nsId, &req, optionFlag) - - registeredStatus = "" + if option != "onlyVm" { + // bring vNet list and register all + inspectedResources, err := InspectResources(connConfig, common.StrVNet) if err != nil { common.CBLog.Error(err) - registeredStatus = " [Failed] " + err.Error() - result.RegisterationOverview.VNet-- - result.RegisterationOverview.Failed++ + result.SystemMessage = err.Error() } - result.RegisterationOutputs.IdList = append(result.RegisterationOutputs.IdList, common.StrVNet+": "+req.Name+registeredStatus) - result.RegisterationOverview.VNet++ - } + for _, r := range inspectedResources.Resources.OnCspOnly.Info { + req := mcir.TbVNetReq{} + req.ConnectionName = connConfig + req.CspVNetId = r.IdByCsp + req.Description = "Ref name: " + r.RefNameOrId + ". CSP managed resource (registered to CB-TB)" + req.Name = req.ConnectionName + "-" + req.CspVNetId + req.Name = strings.ToLower(req.Name) - fmt.Printf("\n\n%s [Elapsed]%s %d \n\n", connConfig, common.StrVNet, int(math.Round(time.Now().Sub(startTime01).Seconds()))) //tmp - startTime02 := time.Now() //tmp + _, err = mcir.CreateVNet(nsId, &req, optionFlag) - // bring SecurityGroup list and register all - inspectedResources, err = InspectResources(connConfig, common.StrSecurityGroup) - if err != nil { - common.CBLog.Error(err) - result.SystemMessage += "//" + err.Error() - } - for _, r := range inspectedResources.Resources.OnCspOnly.Info { - req := mcir.TbSecurityGroupReq{} - req.ConnectionName = connConfig - req.VNetId = "not defined" - req.CspSecurityGroupId = r.IdByCsp - req.Description = "Ref name: " + r.RefNameOrId + ". CSP managed resource (registered to CB-TB)" - req.Name = req.ConnectionName + "-" + req.CspSecurityGroupId - req.Name = strings.ToLower(req.Name) + registeredStatus = "" + if err != nil { + common.CBLog.Error(err) + registeredStatus = " [Failed] " + err.Error() + result.RegisterationOverview.VNet-- + result.RegisterationOverview.Failed++ + } + result.RegisterationOutputs.IdList = append(result.RegisterationOutputs.IdList, common.StrVNet+": "+req.Name+registeredStatus) + result.RegisterationOverview.VNet++ + } - _, err = mcir.CreateSecurityGroup(nsId, &req, optionFlag) + fmt.Printf("\n\n%s [Elapsed]%s %d \n\n", connConfig, common.StrVNet, int(math.Round(time.Now().Sub(startTime01).Seconds()))) //tmp + startTime02 := time.Now() //tmp - registeredStatus = "" + // bring SecurityGroup list and register all + inspectedResources, err = InspectResources(connConfig, common.StrSecurityGroup) if err != nil { common.CBLog.Error(err) - registeredStatus = " [Failed] " + err.Error() - result.RegisterationOverview.SecurityGroup-- - result.RegisterationOverview.Failed++ + result.SystemMessage += "//" + err.Error() + } + for _, r := range inspectedResources.Resources.OnCspOnly.Info { + req := mcir.TbSecurityGroupReq{} + req.ConnectionName = connConfig + req.VNetId = "not defined" + req.CspSecurityGroupId = r.IdByCsp + req.Description = "Ref name: " + r.RefNameOrId + ". CSP managed resource (registered to CB-TB)" + req.Name = req.ConnectionName + "-" + req.CspSecurityGroupId + req.Name = strings.ToLower(req.Name) + + _, err = mcir.CreateSecurityGroup(nsId, &req, optionFlag) + + registeredStatus = "" + if err != nil { + common.CBLog.Error(err) + registeredStatus = " [Failed] " + err.Error() + result.RegisterationOverview.SecurityGroup-- + result.RegisterationOverview.Failed++ + } + result.RegisterationOutputs.IdList = append(result.RegisterationOutputs.IdList, common.StrSecurityGroup+": "+req.Name+registeredStatus) + result.RegisterationOverview.SecurityGroup++ } - result.RegisterationOutputs.IdList = append(result.RegisterationOutputs.IdList, common.StrSecurityGroup+": "+req.Name+registeredStatus) - result.RegisterationOverview.SecurityGroup++ - } - - fmt.Printf("\n\n%s [Elapsed]%s %d \n\n", connConfig, common.StrSecurityGroup, int(math.Round(time.Now().Sub(startTime02).Seconds()))) //tmp - startTime03 := time.Now() //tmp - - // bring SSHKey list and register all - inspectedResources, err = InspectResources(connConfig, common.StrSSHKey) - if err != nil { - common.CBLog.Error(err) - result.SystemMessage += "//" + err.Error() - } - for _, r := range inspectedResources.Resources.OnCspOnly.Info { - req := mcir.TbSshKeyReq{} - req.ConnectionName = connConfig - req.CspSshKeyId = r.IdByCsp - req.Description = "Ref name: " + r.RefNameOrId + ". CSP managed resource (registered to CB-TB)" - req.Name = req.ConnectionName + "-" + req.CspSshKeyId - req.Name = strings.ToLower(req.Name) - - req.Fingerprint = "cannot retrieve" - req.PrivateKey = "cannot retrieve" - req.PublicKey = "cannot retrieve" - req.Username = "cannot retrieve" - _, err = mcir.CreateSshKey(nsId, &req, optionFlag) + fmt.Printf("\n\n%s [Elapsed]%s %d \n\n", connConfig, common.StrSecurityGroup, int(math.Round(time.Now().Sub(startTime02).Seconds()))) //tmp + startTime03 := time.Now() //tmp - registeredStatus = "" + // bring SSHKey list and register all + inspectedResources, err = InspectResources(connConfig, common.StrSSHKey) if err != nil { common.CBLog.Error(err) - registeredStatus = " [Failed] " + err.Error() - result.RegisterationOverview.SshKey-- - result.RegisterationOverview.Failed++ + result.SystemMessage += "//" + err.Error() + } + for _, r := range inspectedResources.Resources.OnCspOnly.Info { + req := mcir.TbSshKeyReq{} + req.ConnectionName = connConfig + req.CspSshKeyId = r.IdByCsp + req.Description = "Ref name: " + r.RefNameOrId + ". CSP managed resource (registered to CB-TB)" + req.Name = req.ConnectionName + "-" + req.CspSshKeyId + req.Name = strings.ToLower(req.Name) + + req.Fingerprint = "cannot retrieve" + req.PrivateKey = "cannot retrieve" + req.PublicKey = "cannot retrieve" + req.Username = "cannot retrieve" + + _, err = mcir.CreateSshKey(nsId, &req, optionFlag) + + registeredStatus = "" + if err != nil { + common.CBLog.Error(err) + registeredStatus = " [Failed] " + err.Error() + result.RegisterationOverview.SshKey-- + result.RegisterationOverview.Failed++ + } + result.RegisterationOutputs.IdList = append(result.RegisterationOutputs.IdList, common.StrSSHKey+": "+req.Name+registeredStatus) + result.RegisterationOverview.SshKey++ } - result.RegisterationOutputs.IdList = append(result.RegisterationOutputs.IdList, common.StrSSHKey+": "+req.Name+registeredStatus) - result.RegisterationOverview.SshKey++ + + fmt.Printf("\n\n%s [Elapsed]%s %d \n\n", connConfig, common.StrSSHKey, int(math.Round(time.Now().Sub(startTime03).Seconds()))) //tmp } - fmt.Printf("\n\n%s [Elapsed]%s %d \n\n", connConfig, common.StrSSHKey, int(math.Round(time.Now().Sub(startTime03).Seconds()))) //tmp - startTime04 := time.Now() //tmp + startTime04 := time.Now() //tmp - // bring VM list and register all - inspectedResources, err = InspectResources(connConfig, common.StrVM) - if err != nil { - common.CBLog.Error(err) - result.SystemMessage += "//" + err.Error() - } - for _, r := range inspectedResources.Resources.OnCspOnly.Info { - req := TbMcisReq{} - req.Description = "MCIS for CSP managed VMs (registered to CB-TB)" - req.InstallMonAgent = "no" - req.Name = mcisId - req.Name = strings.ToLower(req.Name) - - vm := TbVmReq{} - vm.ConnectionName = connConfig - vm.IdByCSP = r.IdByCsp - vm.Description = "Ref name: " + r.RefNameOrId + ". CSP managed VM (registered to CB-TB)" - vm.Name = vm.ConnectionName + "-" + vm.IdByCSP - vm.Name = strings.ToLower(vm.Name) - vm.Label = "not defined" - - vm.ImageId = "cannot retrieve" - vm.SpecId = "cannot retrieve" - vm.SshKeyId = "cannot retrieve" - vm.SubnetId = "cannot retrieve" - vm.VNetId = "cannot retrieve" - vm.SecurityGroupIds = append(vm.SecurityGroupIds, "cannot retrieve") - - req.Vm = append(req.Vm, vm) - - _, err = CreateMcis(nsId, &req, optionFlag) - - registeredStatus = "" + if option != "exceptVm" { + + // bring VM list and register all + inspectedResourcesVm, err := InspectResources(connConfig, common.StrVM) if err != nil { common.CBLog.Error(err) - registeredStatus = " [Failed] " + err.Error() - result.RegisterationOverview.Vm-- - result.RegisterationOverview.Failed++ + result.SystemMessage += "//" + err.Error() + } + for _, r := range inspectedResourcesVm.Resources.OnCspOnly.Info { + req := TbMcisReq{} + req.Description = "MCIS for CSP managed VMs (registered to CB-TB)" + req.InstallMonAgent = "no" + req.Name = mcisId + req.Name = strings.ToLower(req.Name) + + vm := TbVmReq{} + vm.ConnectionName = connConfig + vm.IdByCSP = r.IdByCsp + vm.Description = "Ref name: " + r.RefNameOrId + ". CSP managed VM (registered to CB-TB)" + vm.Name = vm.ConnectionName + "-" + vm.IdByCSP + vm.Name = strings.ToLower(vm.Name) + vm.Label = "not defined" + + vm.ImageId = "cannot retrieve" + vm.SpecId = "cannot retrieve" + vm.SshKeyId = "cannot retrieve" + vm.SubnetId = "cannot retrieve" + vm.VNetId = "cannot retrieve" + vm.SecurityGroupIds = append(vm.SecurityGroupIds, "cannot retrieve") + + req.Vm = append(req.Vm, vm) + + _, err = CreateMcis(nsId, &req, optionFlag) + + registeredStatus = "" + if err != nil { + common.CBLog.Error(err) + registeredStatus = " [Failed] " + err.Error() + result.RegisterationOverview.Vm-- + result.RegisterationOverview.Failed++ + } + result.RegisterationOutputs.IdList = append(result.RegisterationOutputs.IdList, common.StrVM+": "+vm.Name+registeredStatus) + result.RegisterationOverview.Vm++ } - result.RegisterationOutputs.IdList = append(result.RegisterationOutputs.IdList, common.StrVM+": "+vm.Name+registeredStatus) - result.RegisterationOverview.Vm++ } + result.ConnectionName = connConfig result.ElapsedTime = int(math.Round(time.Now().Sub(startTime).Seconds()))