Merge pull request #35 from Skarlso/automatically_update_pipelines

Automatically update pipelines every minute.

Makefile

@@ -4,7 +4,7 @@ GO_LDFLAGS_STATIC=-ldflags "-s -w -extldflags -static"
 default: dev
 
 dev:
-	go run ./cmd/gaia/main.go -homepath=${PWD}/tmp -dev true
+	go run ./cmd/gaia/main.go -homepath=${PWD}/tmp -dev=true -poll=true
 
 compile_frontend:
 	cd ./frontend && \

cmd/gaia/main.go

@@ -43,7 +43,8 @@ func init() {
 	flag.StringVar(&gaia.Cfg.JwtPrivateKeyPath, "jwtPrivateKeyPath", "", "A RSA private key used to sign JWT tokens")
 	flag.BoolVar(&gaia.Cfg.DevMode, "dev", false, "If true, gaia will be started in development mode. Don't use this in production!")
 	flag.BoolVar(&gaia.Cfg.VersionSwitch, "version", false, "If true, will print the version and immediately exit")
-
+	flag.BoolVar(&gaia.Cfg.Poll, "poll", false, "Instead of using a Webhook, keep polling git for changes on pipelines")
+	flag.IntVar(&gaia.Cfg.PVal, "pval", 1, "The interval in minutes in which to poll vcs for changes")
 	// Default values
 	gaia.Cfg.Bolt.Mode = 0600
 }

gaia.go

@@ -91,6 +91,7 @@ type Pipeline struct {
 	SHA256Sum []byte       `json:"sha256sum,omitempty"`
 	Jobs      []Job        `json:"jobs,omitempty"`
 	Created   time.Time    `json:"created,omitempty"`
+	UUID      string       `json:"uuid,omitempty"`
 }
 
 // GitRepo represents a single git repository
@@ -150,6 +151,8 @@ var Cfg *Config
 type Config struct {
 	DevMode           bool
 	VersionSwitch     bool
+	Poll              bool
+	PVal              int
 	ListenPort        string
 	HomePath          string
 	DataPath          string

pipeline/build_golang.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"strings"
 	"time"
 
 	"github.com/gaia-pipeline/gaia"
@@ -40,7 +41,8 @@ func (b *BuildPipelineGolang) PrepareEnvironment(p *gaia.CreatePipeline) error {
 
 	// Set new generated path in pipeline obj for later usage
 	p.Pipeline.Repo.LocalDest = cloneFolder
-	return nil
+	p.Pipeline.UUID = uuid.String()
+	return err
 }
 
 // ExecuteBuild executes the golang build process
@@ -119,6 +121,17 @@ func (b *BuildPipelineGolang) CopyBinary(p *gaia.CreatePipeline) error {
 	return os.Chmod(dest, 0766)
 }
 
+// SavePipeline saves the current pipeline configuration.
+func (b *BuildPipelineGolang) SavePipeline(p *gaia.Pipeline) error {
+	dest := filepath.Join(gaia.Cfg.PipelinePath, appendTypeToName(p.Name, p.Type))
+	p.ExecPath = dest
+	p.Type = gaia.PTypeGolang
+	p.Name = strings.TrimSuffix(filepath.Base(dest), typeDelimiter+gaia.PTypeGolang.String())
+	p.Created = time.Now()
+	// Our pipeline is finished constructing. Save it.
+	return storeService.PipelinePut(p)
+}
+
 // copyFileContents copies the content from source to destination.
 func copyFileContents(src, dst string) (err error) {
 	in, err := os.Open(src)

pipeline/build_golang_test.go

@@ -13,6 +13,7 @@ import (
 	"testing"
 
 	"github.com/gaia-pipeline/gaia"
+	"github.com/gaia-pipeline/gaia/store"
 	hclog "github.com/hashicorp/go-hclog"
 )
 
@@ -111,9 +112,10 @@ func TestExecuteBuildFailPipelineBuild(t *testing.T) {
 	}()
 	gaia.Cfg = new(gaia.Config)
 	gaia.Cfg.HomePath = tmp
+	var logOutput strings.Builder
 	gaia.Cfg.Logger = hclog.New(&hclog.LoggerOptions{
 		Level:  hclog.Trace,
-		Output: hclog.DefaultOutput,
+		Output: &logOutput,
 		Name:   "Gaia",
 	})
 	b := new(BuildPipelineGolang)
@@ -140,9 +142,10 @@ func TestExecuteBuildContextTimeout(t *testing.T) {
 	gaia.Cfg = new(gaia.Config)
 	gaia.Cfg.HomePath = tmp
 	// Initialize shared logger
+	var logOutput strings.Builder
 	gaia.Cfg.Logger = hclog.New(&hclog.LoggerOptions{
 		Level:  hclog.Trace,
-		Output: hclog.DefaultOutput,
+		Output: &logOutput,
 		Name:   "Gaia",
 	})
 	b := new(BuildPipelineGolang)
@@ -161,9 +164,10 @@ func TestExecuteBuildBinaryNotFoundError(t *testing.T) {
 	gaia.Cfg = new(gaia.Config)
 	gaia.Cfg.HomePath = tmp
 	// Initialize shared logger
+	var logOutput strings.Builder
 	gaia.Cfg.Logger = hclog.New(&hclog.LoggerOptions{
 		Level:  hclog.Trace,
-		Output: hclog.DefaultOutput,
+		Output: &logOutput,
 		Name:   "Gaia",
 	})
 	currentPath := os.Getenv("PATH")
@@ -185,9 +189,10 @@ func TestCopyBinary(t *testing.T) {
 	gaia.Cfg = new(gaia.Config)
 	gaia.Cfg.HomePath = tmp
 	// Initialize shared logger
+	var logOutput strings.Builder
 	gaia.Cfg.Logger = hclog.New(&hclog.LoggerOptions{
 		Level:  hclog.Trace,
-		Output: hclog.DefaultOutput,
+		Output: &logOutput,
 		Name:   "Gaia",
 	})
 	b := new(BuildPipelineGolang)
@@ -219,9 +224,10 @@ func TestCopyBinarySrcDoesNotExist(t *testing.T) {
 	gaia.Cfg = new(gaia.Config)
 	gaia.Cfg.HomePath = tmp
 	// Initialize shared logger
+	var logOutput strings.Builder
 	gaia.Cfg.Logger = hclog.New(&hclog.LoggerOptions{
 		Level:  hclog.Trace,
-		Output: hclog.DefaultOutput,
+		Output: &logOutput,
 		Name:   "Gaia",
 	})
 	b := new(BuildPipelineGolang)
@@ -237,3 +243,28 @@ func TestCopyBinarySrcDoesNotExist(t *testing.T) {
 		t.Fatal("a different error occurred then expected: ", err)
 	}
 }
+
+func TestSavePipeline(t *testing.T) {
+	s := store.NewStore()
+	s.Init()
+	storeService = s
+	defer os.Remove("gaia.db")
+	gaia.Cfg = new(gaia.Config)
+	gaia.Cfg.HomePath = "/tmp"
+	gaia.Cfg.PipelinePath = "/tmp/pipelines/"
+	// Initialize shared logger
+	p := new(gaia.Pipeline)
+	p.Name = "main"
+	p.Type = gaia.PTypeGolang
+	b := new(BuildPipelineGolang)
+	err := b.SavePipeline(p)
+	if err != nil {
+		t.Fatal("something went wrong. wasn't supposed to get error: ", err)
+	}
+	if p.Name != "main" {
+		t.Fatal("name of pipeline didn't equal expected 'main'. was instead: ", p.Name)
+	}
+	if p.Type != gaia.PTypeGolang {
+		t.Fatal("type of pipeline was not go. instead was: ", p.Type)
+	}
+}

pipeline/create_pipeline.go

@@ -82,6 +82,15 @@ func CreatePipeline(p *gaia.CreatePipeline) {
 		return
 	}
 
+	// Save the generated pipeline data
+	err = bP.SavePipeline(&p.Pipeline)
+	if err != nil {
+		p.StatusType = gaia.CreatePipelineFailed
+		p.Output = fmt.Sprintf("failed to save the created pipeline: %s", err.Error())
+		storeService.CreatePipelinePut(p)
+		return
+	}
+
 	// Set create pipeline status to complete
 	p.Status = pipelineCompleteStatus
 	p.StatusType = gaia.CreatePipelineSuccess

pipeline/git.go

@@ -2,6 +2,7 @@ package pipeline
 
 import (
 	"strings"
+	"sync"
 
 	"gopkg.in/src-d/go-git.v4/plumbing"
 
@@ -106,3 +107,48 @@ func gitCloneRepo(repo *gaia.GitRepo) error {
 
 	return nil
 }
+
+func updateAllCurrentPipelines() {
+	gaia.Cfg.Logger.Debug("starting updating of pipelines...")
+	var allPipelines []gaia.Pipeline
+	var wg sync.WaitGroup
+	sem := make(chan int, 4)
+	for pipeline := range GlobalActivePipelines.Iter() {
+		allPipelines = append(allPipelines, pipeline)
+	}
+	for _, p := range allPipelines {
+		wg.Add(1)
+		go func(pipe gaia.Pipeline) {
+			defer wg.Done()
+			sem <- 1
+			defer func() { <-sem }()
+			r, err := git.PlainOpen(pipe.Repo.LocalDest)
+			if err != nil {
+				// We don't stop gaia working because of an automated update failed.
+				// So we just move on.
+				gaia.Cfg.Logger.Error("error while opening repo: ", pipe.Repo.LocalDest, err.Error())
+				return
+			}
+			gaia.Cfg.Logger.Debug("checking pipeline: ", pipe.Name)
+			gaia.Cfg.Logger.Debug("selected branch : ", pipe.Repo.SelectedBranch)
+			tree, _ := r.Worktree()
+			err = tree.Pull(&git.PullOptions{
+				RemoteName: "origin",
+			})
+			if err != nil {
+				// It's also an error if the repo is already up to date so we just move on.
+				gaia.Cfg.Logger.Error("error while doing a pull request : ", err.Error())
+				return
+			}
+
+			gaia.Cfg.Logger.Debug("updating pipeline: ", pipe.Name)
+			b := newBuildPipeline(pipe.Type)
+			createPipeline := &gaia.CreatePipeline{}
+			createPipeline.Pipeline = pipe
+			b.ExecuteBuild(createPipeline)
+			b.CopyBinary(createPipeline)
+			gaia.Cfg.Logger.Debug("successfully updated: ", pipe.Name)
+		}(p)
+	}
+	wg.Wait()
+}

pipeline/git_test.go

@@ -2,9 +2,12 @@ package pipeline
 
 import (
 	"os"
+	"strconv"
+	"strings"
 	"testing"
 
 	"github.com/gaia-pipeline/gaia"
+	hclog "github.com/hashicorp/go-hclog"
 )
 
 func TestGitCloneRepo(t *testing.T) {
@@ -19,3 +22,136 @@ func TestGitCloneRepo(t *testing.T) {
 		t.Fatal(err)
 	}
 }
+
+func TestUpdateAllPipelinesRepositoryNotFound(t *testing.T) {
+	tmp := os.TempDir()
+	gaia.Cfg = new(gaia.Config)
+	gaia.Cfg.HomePath = tmp
+	// Initialize shared logger
+	var b strings.Builder
+	gaia.Cfg.Logger = hclog.New(&hclog.LoggerOptions{
+		Level:  hclog.Trace,
+		Output: &b,
+		Name:   "Gaia",
+	})
+
+	p := new(gaia.Pipeline)
+	p.Repo.LocalDest = tmp
+	GlobalActivePipelines = NewActivePipelines()
+	GlobalActivePipelines.Append(*p)
+	updateAllCurrentPipelines()
+	if !strings.Contains(b.String(), "repository does not exist") {
+		t.Fatal("error message not found in logs: ", b.String())
+	}
+}
+
+func TestUpdateAllPipelinesAlreadyUpToDate(t *testing.T) {
+	gaia.Cfg = new(gaia.Config)
+	gaia.Cfg.HomePath = "tmp"
+	// Initialize shared logger
+	var b strings.Builder
+	gaia.Cfg.Logger = hclog.New(&hclog.LoggerOptions{
+		Level:  hclog.Trace,
+		Output: &b,
+		Name:   "Gaia",
+	})
+	repo := &gaia.GitRepo{
+		URL:       "https://github.com/gaia-pipeline/go-test-example",
+		LocalDest: "tmp",
+	}
+	// always ensure that tmp folder is cleaned up
+	defer os.RemoveAll("tmp")
+	err := gitCloneRepo(repo)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	p := new(gaia.Pipeline)
+	p.Name = "main"
+	p.Repo.SelectedBranch = "master"
+	p.Repo.LocalDest = "tmp"
+	GlobalActivePipelines = NewActivePipelines()
+	GlobalActivePipelines.Append(*p)
+	updateAllCurrentPipelines()
+	if !strings.Contains(b.String(), "already up-to-date") {
+		t.Fatal("log output did not contain error message that the repo is up-to-date.: ", b.String())
+	}
+}
+
+func TestUpdateAllPipelinesAlreadyUpToDateWithMoreThanOnePipeline(t *testing.T) {
+	gaia.Cfg = new(gaia.Config)
+	gaia.Cfg.HomePath = "tmp"
+	// Initialize shared logger
+	var b strings.Builder
+	gaia.Cfg.Logger = hclog.New(&hclog.LoggerOptions{
+		Level:  hclog.Trace,
+		Output: &b,
+		Name:   "Gaia",
+	})
+	repo := &gaia.GitRepo{
+		URL:       "https://github.com/gaia-pipeline/go-test-example",
+		LocalDest: "tmp",
+	}
+	// always ensure that tmp folder is cleaned up
+	defer os.RemoveAll("tmp")
+	err := gitCloneRepo(repo)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	p1 := new(gaia.Pipeline)
+	p1.Name = "main"
+	p1.Repo.SelectedBranch = "master"
+	p1.Repo.LocalDest = "tmp"
+	p2 := new(gaia.Pipeline)
+	p2.Name = "main"
+	p2.Repo.SelectedBranch = "master"
+	p2.Repo.LocalDest = "tmp"
+	GlobalActivePipelines = NewActivePipelines()
+	defer func() { GlobalActivePipelines = nil }()
+	GlobalActivePipelines.Append(*p1)
+	GlobalActivePipelines.Append(*p2)
+	updateAllCurrentPipelines()
+	if !strings.Contains(b.String(), "already up-to-date") {
+		t.Fatal("log output did not contain error message that the repo is up-to-date.: ", b.String())
+	}
+}
+
+func TestUpdateAllPipelinesHundredPipelines(t *testing.T) {
+	if _, ok := os.LookupEnv("GAIA_RUN_HUNDRED_PIPELINE_TEST"); !ok {
+		t.Skip()
+	}
+	gaia.Cfg = new(gaia.Config)
+	gaia.Cfg.HomePath = "tmp"
+	// Initialize shared logger
+	var b strings.Builder
+	gaia.Cfg.Logger = hclog.New(&hclog.LoggerOptions{
+		Level:  hclog.Trace,
+		Output: &b,
+		Name:   "Gaia",
+	})
+	repo := &gaia.GitRepo{
+		URL:       "https://github.com/gaia-pipeline/go-test-example",
+		LocalDest: "tmp",
+	}
+	// always ensure that tmp folder is cleaned up
+	defer os.RemoveAll("tmp")
+	err := gitCloneRepo(repo)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	GlobalActivePipelines = NewActivePipelines()
+	for i := 1; i < 100; i++ {
+		p := new(gaia.Pipeline)
+		name := strconv.Itoa(i)
+		p.Name = "main" + name
+		p.Repo.SelectedBranch = "master"
+		p.Repo.LocalDest = "tmp"
+		GlobalActivePipelines.Append(*p)
+	}
+	updateAllCurrentPipelines()
+	if !strings.Contains(b.String(), "already up-to-date") {
+		t.Fatal("log output did not contain error message that the repo is up-to-date.: ", b.String())
+	}
+}