Merge pull request #168 from kthcloud/dev

reworked testing, clean up code, many bug fixes

main.go

@@ -1,7 +1,6 @@
 package main
 
 import (
-	"context"
 	"flag"
 	"go-deploy/pkg/app"
 	"log"
@@ -26,6 +25,7 @@ func main() {
 	_ = flag.Bool("repairer", false, "start repairer")
 	_ = flag.Bool("pinger", false, "start pinger")
 	_ = flag.Bool("snapshotter", false, "start snapshotter")
+	_ = flag.Bool("test-mode", false, "run in test mode")
 
 	flag.Parse()
 
@@ -36,10 +36,19 @@ func main() {
 	repairer := isFlagPassed("repairer")
 	pinger := isFlagPassed("pinger")
 	snapshotter := isFlagPassed("snapshotter")
+	testMode := isFlagPassed("test-mode")
+
+	options := app.Options{
+		Workers:  app.Workers{},
+		TestMode: testMode,
+	}
+
+	if testMode {
+		log.Println("RUNNING IN TEST MODE. NO AUTHENTICATION WILL BE REQUIRED.")
+	}
 
-	var options *app.StartOptions
 	if confirmer || statusUpdater || jobExecutor || repairer || api || pinger || snapshotter {
-		options = &app.StartOptions{
+		options.Workers = app.Workers{
 			API:           api,
 			Confirmer:     confirmer,
 			StatusUpdater: statusUpdater,
@@ -49,26 +58,34 @@ func main() {
 			Snapshotter:   snapshotter,
 		}
 
-		log.Println("api: ", options.API)
-		log.Println("confirmer: ", options.Confirmer)
-		log.Println("status-updater: ", options.StatusUpdater)
-		log.Println("job-executor: ", options.JobExecutor)
-		log.Println("repairer: ", options.Repairer)
-		log.Println("pinger: ", options.Pinger)
-		log.Println("snapshotter: ", options.Snapshotter)
+		workers := &options.Workers
+
+		log.Println("api: ", workers.API)
+		log.Println("confirmer: ", workers.Confirmer)
+		log.Println("status-updater: ", workers.StatusUpdater)
+		log.Println("job-executor: ", workers.JobExecutor)
+		log.Println("repairer: ", workers.Repairer)
+		log.Println("pinger: ", workers.Pinger)
+		log.Println("snapshotter: ", workers.Snapshotter)
 	} else {
+		options.Workers = app.Workers{
+			API:           true,
+			Confirmer:     true,
+			StatusUpdater: true,
+			JobExecutor:   true,
+			Repairer:      true,
+			Pinger:        true,
+			Snapshotter:   true,
+		}
+
 		log.Println("no workers specified, starting all")
 	}
 
-	ctx := context.Background()
-	ctx, cancel := context.WithCancel(ctx)
-	server := app.Start(ctx, options)
-	if server != nil {
-		defer func() {
-			cancel()
-			app.Stop(server)
-		}()
+	deployApp := app.Create(options)
+	if deployApp == nil {
+		log.Fatalln("failed to start app")
 	}
+	defer deployApp.Stop()
 
 	quit := make(chan os.Signal)
 	<-quit

models/dto/body/deployment.go

@@ -13,18 +13,21 @@ type Volume struct {
 	ServerPath string `json:"serverPath" binding:"required,min=1,max=100"`
 }
 
+type GitHub struct {
+	Token        string `json:"token" binding:"required,min=1,max=1000"`
+	RepositoryID int64  `json:"repositoryId" binding:"required"`
+}
+
 type DeploymentCreate struct {
 	Name string `json:"name" binding:"required,rfc1035,min=3,max=30"`
 
+	InternalPort *int     `json:"internalPort" binding:"omitempty,min=1,max=65535"`
 	Private      bool     `json:"private" binding:"omitempty,boolean"`
-	Envs         []Env    `json:"envs" binding:"omitempty,env_list,dive,min=0,max=1000"`
-	Volumes      []Volume `json:"volumes" binding:"omitempty,dive,min=0,max=100"`
-	InitCommands []string `json:"initCommands" binding:"omitempty,dive,min=0,max=100"`
-
-	GitHub *struct {
-		Token        string `json:"token" binding:"required,min=1,max=1000"`
-		RepositoryID int64  `json:"repositoryId" binding:"required"`
-	} `json:"github" binding:"omitempty,dive"`
+	Envs         []Env    `json:"envs" binding:"omitempty,env_list,min=0,max=1000,dive"`
+	Volumes      []Volume `json:"volumes" binding:"omitempty,min=0,max=100,dive"`
+	InitCommands []string `json:"initCommands" binding:"omitempty,min=0,max=100,dive,min=0,max=100"`
+
+	GitHub *GitHub `json:"github" binding:"omitempty,dive"`
 
 	Zone *string `json:"zone" binding:"omitempty"`
 }

models/dto/body/vm.go

@@ -3,7 +3,7 @@ package body
 import "time"
 
 type Port struct {
-	Name         string `json:"name,omitempty" bson:"name" binding:"required"`
+	Name         string `json:"name,omitempty" bson:"name" binding:"required,min=1,max=100"`
 	Port         int    `json:"port,omitempty" bson:"port" binding:"required,min=1,max=65535"`
 	ExternalPort int    `json:"externalPort,omitempty" bson:"externalPort"`
 	Protocol     string `json:"protocol,omitempty" bson:"protocol" binding:"required,oneof=tcp udp"`
@@ -23,21 +23,24 @@ type VmGpu struct {
 }
 
 type VmRead struct {
-	ID               string  `json:"id"`
-	Name             string  `json:"name"`
-	SshPublicKey     string  `json:"sshPublicKey"`
-	Ports            []Port  `json:"ports"`
-	OwnerID          string  `json:"ownerId"`
+	ID      string `json:"id"`
+	Name    string `json:"name"`
+	OwnerID string `json:"ownerId"`
+	Zone    string `json:"zone"`
+
+	Specs        Specs  `json:"specs,omitempty"`
+	Ports        []Port `json:"ports"`
+	GPU          *VmGpu `json:"gpu,omitempty"`
+	SshPublicKey string `json:"sshPublicKey"`
+
 	Status           string  `json:"status"`
 	ConnectionString *string `json:"connectionString,omitempty"`
-	GPU              *VmGpu  `json:"gpu,omitempty"`
-	Specs            Specs   `json:"specs,omitempty"`
 }
 
 type VmCreate struct {
 	Name         string  `json:"name" binding:"required,rfc1035,min=3,max=30"`
 	SshPublicKey string  `json:"sshPublicKey" binding:"required,ssh_public_key"`
-	Ports        []Port  `json:"ports" binding:"omitempty,port_list_names,port_list_numbers,dive,min=0,max=1000"`
+	Ports        []Port  `json:"ports" binding:"omitempty,port_list_names,port_list_numbers,min=0,max=100,dive"`
 	CpuCores     int     `json:"cpuCores" binding:"required,min=2"`
 	RAM          int     `json:"ram" binding:"required,min=1"`
 	DiskSize     int     `json:"diskSize" binding:"required,min=20"`
@@ -67,6 +70,11 @@ type VmUpdated struct {
 	JobID string `json:"jobId"`
 }
 
+type VmSnapshotCreated struct {
+	ID    string `json:"id"`
+	JobID string `json:"jobId"`
+}
+
 type GpuAttached struct {
 	ID    string `json:"id"`
 	JobID string `json:"jobId"`

models/sys/deployment/client.go

@@ -2,8 +2,8 @@ package deployment
 
 import (
 	"go-deploy/models"
-	"go-deploy/models/sys/activityResource"
-	"go-deploy/models/sys/resource"
+	"go-deploy/models/sys/base/activityResource"
+	"go-deploy/models/sys/base/resource"
 	"go.mongodb.org/mongo-driver/mongo"
 )
 

models/sys/deployment/converters.go

@@ -117,8 +117,14 @@ func (p *UpdateParams) FromDTO(dto *body.DeploymentUpdate) {
 	p.InitCommands = dto.InitCommands
 }
 
-func (p *CreateParams) FromDTO(dto *body.DeploymentCreate, fallbackZone *string) {
+func (p *CreateParams) FromDTO(dto *body.DeploymentCreate, fallbackZone *string, fallbackPort int) {
 	p.Name = dto.Name
+
+	if dto.InternalPort == nil {
+		p.InternalPort = fallbackPort
+	} else {
+		p.InternalPort = *dto.InternalPort
+	}
 	p.Private = dto.Private
 	p.Envs = make([]Env, len(dto.Envs))
 	for i, env := range dto.Envs {

models/sys/deployment/deployment.go

@@ -9,19 +9,13 @@ type Deployment struct {
 	Name    string `bson:"name"`
 	OwnerID string `bson:"ownerId"`
 	Zone    string `bson:"zone"`
-
+	
 	CreatedAt   time.Time `bson:"createdAt"`
 	UpdatedAt   time.Time `bson:"updatedAt"`
 	RepairedAt  time.Time `bson:"repairedAt"`
 	RestartedAt time.Time `bson:"restartedAt"`
 	DeletedAt   time.Time `bson:"deletedAt"`
 
-	// keep until migrated
-	Private      bool     `bson:"private"`
-	Envs         []Env    `bson:"envs"`
-	Volumes      []Volume `bson:"volumes"`
-	InitCommands []string `bson:"initCommands"`
-
 	Activities []string `bson:"activities"`
 
 	Apps       map[string]App `bson:"apps"`

models/sys/deployment/helpers.go

@@ -16,6 +16,7 @@ func (client *Client) Create(deploymentID, ownerID string, params *CreateParams)
 	appName := "main"
 	mainApp := App{
 		Name:         appName,
+		InternalPort: params.InternalPort,
 		Private:      params.Private,
 		Envs:         params.Envs,
 		Volumes:      params.Volumes,
@@ -25,20 +26,16 @@ func (client *Client) Create(deploymentID, ownerID string, params *CreateParams)
 	}
 
 	deployment := Deployment{
-		ID:           deploymentID,
-		Name:         params.Name,
-		OwnerID:      ownerID,
-		Zone:         params.Zone,
-		CreatedAt:    time.Now(),
-		UpdatedAt:    time.Time{},
-		RepairedAt:   time.Time{},
-		RestartedAt:  time.Time{},
-		Private:      false,
-		Envs:         make([]Env, 0),
-		Volumes:      make([]Volume, 0),
-		InitCommands: make([]string, 0),
-		Apps:         map[string]App{appName: mainApp},
-		Activities:   []string{ActivityBeingCreated},
+		ID:          deploymentID,
+		Name:        params.Name,
+		OwnerID:     ownerID,
+		Zone:        params.Zone,
+		CreatedAt:   time.Now(),
+		UpdatedAt:   time.Time{},
+		RepairedAt:  time.Time{},
+		RestartedAt: time.Time{},
+		Apps:        map[string]App{appName: mainApp},
+		Activities:  []string{ActivityBeingCreated},
 		Subsystems: Subsystems{
 			GitLab: subsystems.GitLab{
 				LastBuild: subsystems.GitLabBuild{
@@ -142,6 +139,10 @@ func (client *Client) UpdateWithParamsByID(id string, update *UpdateParams) erro
 		mainApp.ExtraDomains = *update.ExtraDomains
 	}
 
+	if update.Volumes != nil {
+		mainApp.Volumes = *update.Volumes
+	}
+
 	deployment.Apps["main"] = *mainApp
 
 	_, err = client.Collection.UpdateOne(context.TODO(),

models/sys/deployment/models.go

@@ -4,6 +4,7 @@ import "go-deploy/models/sys/deployment/subsystems"
 
 type App struct {
 	Name         string   `bson:"name"`
+	InternalPort int      `bson:"internalPort"`
 	Private      bool     `bson:"private"`
 	Envs         []Env    `bson:"envs"`
 	Volumes      []Volume `bson:"volumes"`
@@ -49,13 +50,17 @@ type GitHubCreateParams struct {
 }
 
 type CreateParams struct {
-	Name         string              `json:"name" bson:"name"`
-	Private      bool                `json:"private" bson:"private"`
-	Envs         []Env               `json:"envs" bson:"envs"`
-	Volumes      []Volume            `json:"volumes" bson:"volumes"`
-	InitCommands []string            `json:"initCommands" bson:"initCommands"`
-	GitHub       *GitHubCreateParams `json:"github,omitempty" bson:"github,omitempty"`
-	Zone         string              `json:"zone,omitempty" bson:"zoneId,omitempty"`
+	Name string `json:"name" bson:"name"`
+
+	InternalPort int      `json:"internalPort" bson:"internalPort"`
+	Private      bool     `json:"private" bson:"private"`
+	Envs         []Env    `json:"envs" bson:"envs"`
+	Volumes      []Volume `json:"volumes" bson:"volumes"`
+	InitCommands []string `json:"initCommands" bson:"initCommands"`
+
+	GitHub *GitHubCreateParams `json:"github,omitempty" bson:"github,omitempty"`
+
+	Zone string `json:"zone,omitempty" bson:"zoneId,omitempty"`
 }
 
 type BuildParams struct {

models/sys/enviroment/models.go

@@ -124,7 +124,7 @@ type Environment struct {
 	} `yaml:"landing"`
 
 	Harbor struct {
-		Url           string `yaml:"url"`
+		URL           string `yaml:"url"`
 		User          string `yaml:"user"`
 		Password      string `yaml:"password"`
 		WebhookSecret string `yaml:"webhookSecret"`

models/sys/job/client.go

@@ -2,7 +2,7 @@ package job
 
 import (
 	"go-deploy/models"
-	"go-deploy/models/sys/resource"
+	"go-deploy/models/sys/base/resource"
 )
 
 type Client struct {

models/sys/user/client.go

@@ -2,7 +2,7 @@ package user
 
 import (
 	"go-deploy/models"
-	"go-deploy/models/sys/resource"
+	"go-deploy/models/sys/base/resource"
 )
 
 type Client struct {

models/sys/vm/client.go

@@ -2,8 +2,8 @@ package vm
 
 import (
 	"go-deploy/models"
-	"go-deploy/models/sys/activityResource"
-	"go-deploy/models/sys/resource"
+	"go-deploy/models/sys/base/activityResource"
+	"go-deploy/models/sys/base/resource"
 	"go.mongodb.org/mongo-driver/mongo"
 )
 

models/sys/vm/converters.go

@@ -36,19 +36,20 @@ func (vm *VM) ToDTO(status string, connectionString *string, gpu *body.GpuRead,
 	}
 
 	return body.VmRead{
-		ID:               vm.ID,
-		Name:             vm.Name,
-		SshPublicKey:     vm.SshPublicKey,
-		Ports:            ports,
-		OwnerID:          vm.OwnerID,
-		Status:           status,
-		ConnectionString: connectionString,
-		GPU:              vmGpu,
+		ID:      vm.ID,
+		Name:    vm.Name,
+		OwnerID: vm.OwnerID,
+		Zone:    vm.Zone,
 		Specs: body.Specs{
 			CpuCores: vm.Specs.CpuCores,
 			RAM:      vm.Specs.RAM,
 			DiskSize: vm.Specs.DiskSize,
 		},
+		Ports:            ports,
+		GPU:              vmGpu,
+		SshPublicKey:     vm.SshPublicKey,
+		Status:           status,
+		ConnectionString: connectionString,
 	}
 }
 

models/sys/vm/gpu/client.go

@@ -2,7 +2,7 @@ package gpu
 
 import (
 	"go-deploy/models"
-	"go-deploy/models/sys/resource"
+	"go-deploy/models/sys/base/resource"
 )
 
 type Client struct {