diff --git a/Makefile b/Makefile index c063e9b1..aeb769a7 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ GO_LDFLAGS_STATIC=-ldflags "-s -w -extldflags -static" default: dev dev: - go run ./cmd/gaia/main.go -homepath=${PWD}/tmp -dev true + go run ./cmd/gaia/main.go -homepath=${PWD}/tmp -dev=true -poll=true compile_frontend: cd ./frontend && \ diff --git a/cmd/gaia/main.go b/cmd/gaia/main.go index 08faa46a..d304e4bd 100644 --- a/cmd/gaia/main.go +++ b/cmd/gaia/main.go @@ -43,7 +43,8 @@ func init() { flag.StringVar(&gaia.Cfg.JwtPrivateKeyPath, "jwtPrivateKeyPath", "", "A RSA private key used to sign JWT tokens") flag.BoolVar(&gaia.Cfg.DevMode, "dev", false, "If true, gaia will be started in development mode. Don't use this in production!") flag.BoolVar(&gaia.Cfg.VersionSwitch, "version", false, "If true, will print the version and immediately exit") - + flag.BoolVar(&gaia.Cfg.Poll, "poll", false, "Instead of using a Webhook, keep polling git for changes on pipelines") + flag.IntVar(&gaia.Cfg.PVal, "pval", 1, "The interval in minutes in which to poll vcs for changes") // Default values gaia.Cfg.Bolt.Mode = 0600 } diff --git a/gaia.go b/gaia.go index 32171d7c..d6ad2092 100644 --- a/gaia.go +++ b/gaia.go @@ -91,6 +91,7 @@ type Pipeline struct { SHA256Sum []byte `json:"sha256sum,omitempty"` Jobs []Job `json:"jobs,omitempty"` Created time.Time `json:"created,omitempty"` + UUID string `json:"uuid,omitempty"` } // GitRepo represents a single git repository @@ -150,6 +151,8 @@ var Cfg *Config type Config struct { DevMode bool VersionSwitch bool + Poll bool + PVal int ListenPort string HomePath string DataPath string diff --git a/pipeline/build_golang.go b/pipeline/build_golang.go index 80b2968e..66efe07a 100644 --- a/pipeline/build_golang.go +++ b/pipeline/build_golang.go @@ -6,6 +6,7 @@ import ( "os" "os/exec" "path/filepath" + "strings" "time" "github.com/gaia-pipeline/gaia" @@ -40,7 +41,8 @@ func (b *BuildPipelineGolang) PrepareEnvironment(p *gaia.CreatePipeline) error { // Set new generated path in pipeline obj for later usage p.Pipeline.Repo.LocalDest = cloneFolder - return nil + p.Pipeline.UUID = uuid.String() + return err } // ExecuteBuild executes the golang build process @@ -119,6 +121,17 @@ func (b *BuildPipelineGolang) CopyBinary(p *gaia.CreatePipeline) error { return os.Chmod(dest, 0766) } +// SavePipeline saves the current pipeline configuration. +func (b *BuildPipelineGolang) SavePipeline(p *gaia.Pipeline) error { + dest := filepath.Join(gaia.Cfg.PipelinePath, appendTypeToName(p.Name, p.Type)) + p.ExecPath = dest + p.Type = gaia.PTypeGolang + p.Name = strings.TrimSuffix(filepath.Base(dest), typeDelimiter+gaia.PTypeGolang.String()) + p.Created = time.Now() + // Our pipeline is finished constructing. Save it. + return storeService.PipelinePut(p) +} + // copyFileContents copies the content from source to destination. func copyFileContents(src, dst string) (err error) { in, err := os.Open(src) diff --git a/pipeline/build_golang_test.go b/pipeline/build_golang_test.go index 87739f3d..7ba8c30b 100644 --- a/pipeline/build_golang_test.go +++ b/pipeline/build_golang_test.go @@ -13,6 +13,7 @@ import ( "testing" "github.com/gaia-pipeline/gaia" + "github.com/gaia-pipeline/gaia/store" hclog "github.com/hashicorp/go-hclog" ) @@ -111,9 +112,10 @@ func TestExecuteBuildFailPipelineBuild(t *testing.T) { }() gaia.Cfg = new(gaia.Config) gaia.Cfg.HomePath = tmp + var logOutput strings.Builder gaia.Cfg.Logger = hclog.New(&hclog.LoggerOptions{ Level: hclog.Trace, - Output: hclog.DefaultOutput, + Output: &logOutput, Name: "Gaia", }) b := new(BuildPipelineGolang) @@ -140,9 +142,10 @@ func TestExecuteBuildContextTimeout(t *testing.T) { gaia.Cfg = new(gaia.Config) gaia.Cfg.HomePath = tmp // Initialize shared logger + var logOutput strings.Builder gaia.Cfg.Logger = hclog.New(&hclog.LoggerOptions{ Level: hclog.Trace, - Output: hclog.DefaultOutput, + Output: &logOutput, Name: "Gaia", }) b := new(BuildPipelineGolang) @@ -161,9 +164,10 @@ func TestExecuteBuildBinaryNotFoundError(t *testing.T) { gaia.Cfg = new(gaia.Config) gaia.Cfg.HomePath = tmp // Initialize shared logger + var logOutput strings.Builder gaia.Cfg.Logger = hclog.New(&hclog.LoggerOptions{ Level: hclog.Trace, - Output: hclog.DefaultOutput, + Output: &logOutput, Name: "Gaia", }) currentPath := os.Getenv("PATH") @@ -185,9 +189,10 @@ func TestCopyBinary(t *testing.T) { gaia.Cfg = new(gaia.Config) gaia.Cfg.HomePath = tmp // Initialize shared logger + var logOutput strings.Builder gaia.Cfg.Logger = hclog.New(&hclog.LoggerOptions{ Level: hclog.Trace, - Output: hclog.DefaultOutput, + Output: &logOutput, Name: "Gaia", }) b := new(BuildPipelineGolang) @@ -219,9 +224,10 @@ func TestCopyBinarySrcDoesNotExist(t *testing.T) { gaia.Cfg = new(gaia.Config) gaia.Cfg.HomePath = tmp // Initialize shared logger + var logOutput strings.Builder gaia.Cfg.Logger = hclog.New(&hclog.LoggerOptions{ Level: hclog.Trace, - Output: hclog.DefaultOutput, + Output: &logOutput, Name: "Gaia", }) b := new(BuildPipelineGolang) @@ -237,3 +243,28 @@ func TestCopyBinarySrcDoesNotExist(t *testing.T) { t.Fatal("a different error occurred then expected: ", err) } } + +func TestSavePipeline(t *testing.T) { + s := store.NewStore() + s.Init() + storeService = s + defer os.Remove("gaia.db") + gaia.Cfg = new(gaia.Config) + gaia.Cfg.HomePath = "/tmp" + gaia.Cfg.PipelinePath = "/tmp/pipelines/" + // Initialize shared logger + p := new(gaia.Pipeline) + p.Name = "main" + p.Type = gaia.PTypeGolang + b := new(BuildPipelineGolang) + err := b.SavePipeline(p) + if err != nil { + t.Fatal("something went wrong. wasn't supposed to get error: ", err) + } + if p.Name != "main" { + t.Fatal("name of pipeline didn't equal expected 'main'. was instead: ", p.Name) + } + if p.Type != gaia.PTypeGolang { + t.Fatal("type of pipeline was not go. instead was: ", p.Type) + } +} diff --git a/pipeline/create_pipeline.go b/pipeline/create_pipeline.go index 8b77a264..aebaab75 100644 --- a/pipeline/create_pipeline.go +++ b/pipeline/create_pipeline.go @@ -82,6 +82,15 @@ func CreatePipeline(p *gaia.CreatePipeline) { return } + // Save the generated pipeline data + err = bP.SavePipeline(&p.Pipeline) + if err != nil { + p.StatusType = gaia.CreatePipelineFailed + p.Output = fmt.Sprintf("failed to save the created pipeline: %s", err.Error()) + storeService.CreatePipelinePut(p) + return + } + // Set create pipeline status to complete p.Status = pipelineCompleteStatus p.StatusType = gaia.CreatePipelineSuccess diff --git a/pipeline/git.go b/pipeline/git.go index 02188c7f..3cff6843 100644 --- a/pipeline/git.go +++ b/pipeline/git.go @@ -2,6 +2,7 @@ package pipeline import ( "strings" + "sync" "gopkg.in/src-d/go-git.v4/plumbing" @@ -106,3 +107,48 @@ func gitCloneRepo(repo *gaia.GitRepo) error { return nil } + +func updateAllCurrentPipelines() { + gaia.Cfg.Logger.Debug("starting updating of pipelines...") + var allPipelines []gaia.Pipeline + var wg sync.WaitGroup + sem := make(chan int, 4) + for pipeline := range GlobalActivePipelines.Iter() { + allPipelines = append(allPipelines, pipeline) + } + for _, p := range allPipelines { + wg.Add(1) + go func(pipe gaia.Pipeline) { + defer wg.Done() + sem <- 1 + defer func() { <-sem }() + r, err := git.PlainOpen(pipe.Repo.LocalDest) + if err != nil { + // We don't stop gaia working because of an automated update failed. + // So we just move on. + gaia.Cfg.Logger.Error("error while opening repo: ", pipe.Repo.LocalDest, err.Error()) + return + } + gaia.Cfg.Logger.Debug("checking pipeline: ", pipe.Name) + gaia.Cfg.Logger.Debug("selected branch : ", pipe.Repo.SelectedBranch) + tree, _ := r.Worktree() + err = tree.Pull(&git.PullOptions{ + RemoteName: "origin", + }) + if err != nil { + // It's also an error if the repo is already up to date so we just move on. + gaia.Cfg.Logger.Error("error while doing a pull request : ", err.Error()) + return + } + + gaia.Cfg.Logger.Debug("updating pipeline: ", pipe.Name) + b := newBuildPipeline(pipe.Type) + createPipeline := &gaia.CreatePipeline{} + createPipeline.Pipeline = pipe + b.ExecuteBuild(createPipeline) + b.CopyBinary(createPipeline) + gaia.Cfg.Logger.Debug("successfully updated: ", pipe.Name) + }(p) + } + wg.Wait() +} diff --git a/pipeline/git_test.go b/pipeline/git_test.go index 23fb22c2..31af1aee 100644 --- a/pipeline/git_test.go +++ b/pipeline/git_test.go @@ -2,9 +2,12 @@ package pipeline import ( "os" + "strconv" + "strings" "testing" "github.com/gaia-pipeline/gaia" + hclog "github.com/hashicorp/go-hclog" ) func TestGitCloneRepo(t *testing.T) { @@ -19,3 +22,136 @@ func TestGitCloneRepo(t *testing.T) { t.Fatal(err) } } + +func TestUpdateAllPipelinesRepositoryNotFound(t *testing.T) { + tmp := os.TempDir() + gaia.Cfg = new(gaia.Config) + gaia.Cfg.HomePath = tmp + // Initialize shared logger + var b strings.Builder + gaia.Cfg.Logger = hclog.New(&hclog.LoggerOptions{ + Level: hclog.Trace, + Output: &b, + Name: "Gaia", + }) + + p := new(gaia.Pipeline) + p.Repo.LocalDest = tmp + GlobalActivePipelines = NewActivePipelines() + GlobalActivePipelines.Append(*p) + updateAllCurrentPipelines() + if !strings.Contains(b.String(), "repository does not exist") { + t.Fatal("error message not found in logs: ", b.String()) + } +} + +func TestUpdateAllPipelinesAlreadyUpToDate(t *testing.T) { + gaia.Cfg = new(gaia.Config) + gaia.Cfg.HomePath = "tmp" + // Initialize shared logger + var b strings.Builder + gaia.Cfg.Logger = hclog.New(&hclog.LoggerOptions{ + Level: hclog.Trace, + Output: &b, + Name: "Gaia", + }) + repo := &gaia.GitRepo{ + URL: "https://github.com/gaia-pipeline/go-test-example", + LocalDest: "tmp", + } + // always ensure that tmp folder is cleaned up + defer os.RemoveAll("tmp") + err := gitCloneRepo(repo) + if err != nil { + t.Fatal(err) + } + + p := new(gaia.Pipeline) + p.Name = "main" + p.Repo.SelectedBranch = "master" + p.Repo.LocalDest = "tmp" + GlobalActivePipelines = NewActivePipelines() + GlobalActivePipelines.Append(*p) + updateAllCurrentPipelines() + if !strings.Contains(b.String(), "already up-to-date") { + t.Fatal("log output did not contain error message that the repo is up-to-date.: ", b.String()) + } +} + +func TestUpdateAllPipelinesAlreadyUpToDateWithMoreThanOnePipeline(t *testing.T) { + gaia.Cfg = new(gaia.Config) + gaia.Cfg.HomePath = "tmp" + // Initialize shared logger + var b strings.Builder + gaia.Cfg.Logger = hclog.New(&hclog.LoggerOptions{ + Level: hclog.Trace, + Output: &b, + Name: "Gaia", + }) + repo := &gaia.GitRepo{ + URL: "https://github.com/gaia-pipeline/go-test-example", + LocalDest: "tmp", + } + // always ensure that tmp folder is cleaned up + defer os.RemoveAll("tmp") + err := gitCloneRepo(repo) + if err != nil { + t.Fatal(err) + } + + p1 := new(gaia.Pipeline) + p1.Name = "main" + p1.Repo.SelectedBranch = "master" + p1.Repo.LocalDest = "tmp" + p2 := new(gaia.Pipeline) + p2.Name = "main" + p2.Repo.SelectedBranch = "master" + p2.Repo.LocalDest = "tmp" + GlobalActivePipelines = NewActivePipelines() + defer func() { GlobalActivePipelines = nil }() + GlobalActivePipelines.Append(*p1) + GlobalActivePipelines.Append(*p2) + updateAllCurrentPipelines() + if !strings.Contains(b.String(), "already up-to-date") { + t.Fatal("log output did not contain error message that the repo is up-to-date.: ", b.String()) + } +} + +func TestUpdateAllPipelinesHundredPipelines(t *testing.T) { + if _, ok := os.LookupEnv("GAIA_RUN_HUNDRED_PIPELINE_TEST"); !ok { + t.Skip() + } + gaia.Cfg = new(gaia.Config) + gaia.Cfg.HomePath = "tmp" + // Initialize shared logger + var b strings.Builder + gaia.Cfg.Logger = hclog.New(&hclog.LoggerOptions{ + Level: hclog.Trace, + Output: &b, + Name: "Gaia", + }) + repo := &gaia.GitRepo{ + URL: "https://github.com/gaia-pipeline/go-test-example", + LocalDest: "tmp", + } + // always ensure that tmp folder is cleaned up + defer os.RemoveAll("tmp") + err := gitCloneRepo(repo) + if err != nil { + t.Fatal(err) + } + + GlobalActivePipelines = NewActivePipelines() + for i := 1; i < 100; i++ { + p := new(gaia.Pipeline) + name := strconv.Itoa(i) + p.Name = "main" + name + p.Repo.SelectedBranch = "master" + p.Repo.LocalDest = "tmp" + GlobalActivePipelines.Append(*p) + } + updateAllCurrentPipelines() + if !strings.Contains(b.String(), "already up-to-date") { + t.Fatal("log output did not contain error message that the repo is up-to-date.: ", b.String()) + } +} diff --git a/pipeline/pipeline.go b/pipeline/pipeline.go index 65e6ebb8..1497fb31 100644 --- a/pipeline/pipeline.go +++ b/pipeline/pipeline.go @@ -22,6 +22,9 @@ type BuildPipeline interface { // CopyBinary copies the result from the compile process // to the plugins folder. CopyBinary(*gaia.CreatePipeline) error + + // SavePipeline the pipeline in its current format + SavePipeline(*gaia.Pipeline) error } // ActivePipelines holds all active pipelines. diff --git a/pipeline/ticker.go b/pipeline/ticker.go index 09718730..dabf8f08 100644 --- a/pipeline/ticker.go +++ b/pipeline/ticker.go @@ -3,6 +3,7 @@ package pipeline import ( "bytes" "crypto/sha256" + "fmt" "io" "io/ioutil" "os" @@ -52,6 +53,24 @@ func InitTicker(store *store.Store, scheduler *scheduler.Scheduler) { } } }() + + if gaia.Cfg.Poll { + if gaia.Cfg.PVal < 1 || gaia.Cfg.PVal > 99 { + errorMessage := fmt.Sprintf("Invalid value defined for poll interval. Will be using default of 1. Value was: %d, should be between 1-99.", gaia.Cfg.PVal) + gaia.Cfg.Logger.Info(errorMessage) + gaia.Cfg.PVal = 1 + } + pollTicket := time.NewTicker(time.Duration(gaia.Cfg.PVal) * time.Minute) + go func() { + defer pollTicket.Stop() + for { + select { + case <-pollTicket.C: + updateAllCurrentPipelines() + } + } + }() + } } // checkActivePipelines looks up all files in the pipeline folder. @@ -116,18 +135,15 @@ func checkActivePipelines() { continue } - // We couldn't find the pipeline. Create a new one. - var shouldStore = false + // Pipeline is a drop-in build. Set up a template for it. + shouldStore := false if pipeline == nil { - // Create pipeline object and fill it with information pipeline = &gaia.Pipeline{ Name: pName, Type: pType, ExecPath: filepath.Join(gaia.Cfg.PipelinePath, file.Name()), Created: time.Now(), } - - // We should store it shouldStore = true } @@ -142,7 +158,7 @@ func checkActivePipelines() { // Let us try to start the plugin and receive all implemented jobs schedulerService.SetPipelineJobs(pipeline) - // Put pipeline into store only when it was new created. + // We encountered a drop-in pipeline previously. Now is the time to save it. if shouldStore { storeService.PipelinePut(pipeline) }