Global atlantis lock new release branch (#1473)

* Orca 679 global atlantis lock new release branch (#49)

* Adding CommandLocker to boltDB and exposing it through locker interface

* Apply lock ui and apply command lock controller

* Minor comments

* Adding more tests and refactorinng

* Linting fixes

* creating applyLockingClient variable to fix interface error

* nullsink for stats

* Addressing PR comments

* fixing e2e tests

* linting fix fml

* Update outdated function descriptions

Address PR comments

* revert stats sink changes

* remove unnecessary dependencies

server/events/apply_command_runner.go

@@ -2,14 +2,15 @@ package events
 
 import (
 	"github.com/runatlantis/atlantis/server/events/db"
+	"github.com/runatlantis/atlantis/server/events/locking"
 	"github.com/runatlantis/atlantis/server/events/models"
 	"github.com/runatlantis/atlantis/server/events/vcs"
 )
 
 func NewApplyCommandRunner(
 	vcsClient vcs.Client,
 	disableApplyAll bool,
-	disableApply bool,
+	applyCommandLocker locking.ApplyLockChecker,
 	commitStatusUpdater CommitStatusUpdater,
 	prjCommandBuilder ProjectApplyCommandBuilder,
 	prjCmdRunner ProjectApplyCommandRunner,
@@ -22,7 +23,7 @@ func NewApplyCommandRunner(
 	return &ApplyCommandRunner{
 		vcsClient:           vcsClient,
 		DisableApplyAll:     disableApplyAll,
-		DisableApply:        disableApply,
+		locker:              applyCommandLocker,
 		commitStatusUpdater: commitStatusUpdater,
 		prjCmdBuilder:       prjCommandBuilder,
 		prjCmdRunner:        prjCmdRunner,
@@ -36,8 +37,8 @@ func NewApplyCommandRunner(
 
 type ApplyCommandRunner struct {
 	DisableApplyAll     bool
-	DisableApply        bool
 	DB                  *db.BoltDB
+	locker              locking.ApplyLockChecker
 	vcsClient           vcs.Client
 	commitStatusUpdater CommitStatusUpdater
 	prjCmdBuilder       ProjectApplyCommandBuilder
@@ -53,7 +54,15 @@ func (a *ApplyCommandRunner) Run(ctx *CommandContext, cmd *CommentCommand) {
 	baseRepo := ctx.Pull.BaseRepo
 	pull := ctx.Pull
 
-	if a.DisableApply {
+	locked, err := a.IsLocked()
+	// CheckApplyLock falls back to DisableApply flag if fetching the lock
+	// raises an erro r
+	// We will log failure as warning
+	if err != nil {
+		ctx.Log.Warn("checking global apply lock: %s", err)
+	}
+
+	if locked {
 		ctx.Log.Info("ignoring apply command since apply disabled globally")
 		if err := a.vcsClient.CreateComment(baseRepo, pull.Num, applyDisabledComment, models.ApplyCommand.String()); err != nil {
 			ctx.Log.Err("unable to comment on pull request: %s", err)
@@ -135,6 +144,12 @@ func (a *ApplyCommandRunner) Run(ctx *CommandContext, cmd *CommentCommand) {
 	}
 }
 
+func (a *ApplyCommandRunner) IsLocked() (bool, error) {
+	lock, err := a.locker.CheckApplyLock()
+
+	return lock.Locked, err
+}
+
 func (a *ApplyCommandRunner) isParallelEnabled(projectCmds []models.ProjectCommandContext) bool {
 	return len(projectCmds) > 0 && projectCmds[0].ParallelApplyEnabled
 }

server/events/apply_command_runner_test.go

@@ -0,0 +1,69 @@
+package events_test
+
+import (
+	"errors"
+	"testing"
+
+	"github.com/google/go-github/v31/github"
+	. "github.com/petergtz/pegomock"
+	"github.com/runatlantis/atlantis/server/events"
+	"github.com/runatlantis/atlantis/server/events/locking"
+	"github.com/runatlantis/atlantis/server/events/models"
+	"github.com/runatlantis/atlantis/server/events/models/fixtures"
+)
+
+func TestApplyCommandRunner_IsLocked(t *testing.T) {
+	RegisterMockTestingT(t)
+
+	cases := []struct {
+		Description    string
+		ApplyLocked    bool
+		ApplyLockError error
+		ExpComment     string
+	}{
+		{
+			Description:    "When global apply lock is present IsDisabled returns true",
+			ApplyLocked:    true,
+			ApplyLockError: nil,
+			ExpComment:     "**Error:** Running `atlantis apply` is disabled.",
+		},
+		{
+			Description:    "When no global apply lock is present and DisableApply flag is false IsDisabled returns false",
+			ApplyLocked:    false,
+			ApplyLockError: nil,
+			ExpComment:     "Ran Apply for 0 projects:\n\n\n\n",
+		},
+		{
+			Description:    "If ApplyLockChecker returns an error IsDisabled return value of DisableApply flag",
+			ApplyLockError: errors.New("error"),
+			ApplyLocked:    false,
+			ExpComment:     "Ran Apply for 0 projects:\n\n\n\n",
+		},
+	}
+
+	for _, c := range cases {
+		t.Run(c.Description, func(t *testing.T) {
+			vcsClient := setup(t)
+
+			pull := &github.PullRequest{
+				State: github.String("open"),
+			}
+			modelPull := models.PullRequest{BaseRepo: fixtures.GithubRepo, State: models.OpenPullState, Num: fixtures.Pull.Num}
+			When(githubGetter.GetPullRequest(fixtures.GithubRepo, fixtures.Pull.Num)).ThenReturn(pull, nil)
+			When(eventParsing.ParseGithubPull(pull)).ThenReturn(modelPull, modelPull.BaseRepo, fixtures.GithubRepo, nil)
+
+			ctx := &events.CommandContext{
+				User:     fixtures.User,
+				Log:      noopLogger,
+				Pull:     modelPull,
+				HeadRepo: fixtures.GithubRepo,
+				Trigger:  events.Comment,
+			}
+
+			When(applyLockChecker.CheckApplyLock()).ThenReturn(locking.ApplyCommandLock{Locked: c.ApplyLocked}, c.ApplyLockError)
+			applyCommandRunner.Run(ctx, &events.CommentCommand{Name: models.ApplyCommand})
+
+			vcsClient.VerifyWasCalledOnce().CreateComment(fixtures.GithubRepo, modelPull.Num, c.ExpComment, "apply")
+		})
+	}
+}

server/events/command_runner_test.go

@@ -26,6 +26,7 @@ import (
 	"github.com/google/go-github/v31/github"
 	. "github.com/petergtz/pegomock"
 	"github.com/runatlantis/atlantis/server/events"
+	lockingmocks "github.com/runatlantis/atlantis/server/events/locking/mocks"
 	"github.com/runatlantis/atlantis/server/events/mocks"
 	eventmocks "github.com/runatlantis/atlantis/server/events/mocks"
 	"github.com/runatlantis/atlantis/server/events/mocks/matchers"
@@ -59,6 +60,7 @@ var autoMerger *events.AutoMerger
 var policyCheckCommandRunner *events.PolicyCheckCommandRunner
 var approvePoliciesCommandRunner *events.ApprovePoliciesCommandRunner
 var planCommandRunner *events.PlanCommandRunner
+var applyLockChecker *lockingmocks.MockApplyLockChecker
 var applyCommandRunner *events.ApplyCommandRunner
 var unlockCommandRunner *events.UnlockCommandRunner
 var preWorkflowHooksCommandRunner events.PreWorkflowHooksCommandRunner
@@ -85,6 +87,7 @@ func setup(t *testing.T) *vcsmocks.MockClient {
 
 	drainer = &events.Drainer{}
 	deleteLockCommand = eventmocks.NewMockDeleteLockCommand()
+	applyLockChecker = lockingmocks.NewMockApplyLockChecker()
 	When(logger.GetLevel()).ThenReturn(logging.Info)
 	When(logger.NewLogger("runatlantis/atlantis#1", true, logging.Info)).
 		ThenReturn(pullLogger)
@@ -131,7 +134,7 @@ func setup(t *testing.T) *vcsmocks.MockClient {
 	applyCommandRunner = events.NewApplyCommandRunner(
 		vcsClient,
 		false,
-		false,
+		applyLockChecker,
 		commitUpdater,
 		projectCommandBuilder,
 		projectCommandRunner,
@@ -292,23 +295,6 @@ func TestRunCommentCommand_DisableApplyAllDisabled(t *testing.T) {
 	vcsClient.VerifyWasCalledOnce().CreateComment(fixtures.GithubRepo, modelPull.Num, "**Error:** Running `atlantis apply` without flags is disabled. You must specify which project to apply via the `-d <dir>`, `-w <workspace>` or `-p <project name>` flags.", "apply")
 }
 
-func TestRunCommentCommand_ApplyDisabled(t *testing.T) {
-	t.Log("if \"atlantis apply\" is run and this is disabled globally atlantis should" +
-		" comment saying that this is not allowed")
-	vcsClient := setup(t)
-	applyCommandRunner.DisableApply = true
-	defer func() { applyCommandRunner.DisableApply = false }()
-	pull := &github.PullRequest{
-		State: github.String("open"),
-	}
-	modelPull := models.PullRequest{BaseRepo: fixtures.GithubRepo, State: models.OpenPullState, Num: fixtures.Pull.Num}
-	When(githubGetter.GetPullRequest(fixtures.GithubRepo, fixtures.Pull.Num)).ThenReturn(pull, nil)
-	When(eventParsing.ParseGithubPull(pull)).ThenReturn(modelPull, modelPull.BaseRepo, fixtures.GithubRepo, nil)
-
-	ch.RunCommentCommand(fixtures.GithubRepo, nil, nil, fixtures.User, modelPull.Num, &events.CommentCommand{Name: models.ApplyCommand})
-	vcsClient.VerifyWasCalledOnce().CreateComment(fixtures.GithubRepo, modelPull.Num, "**Error:** Running `atlantis apply` is disabled.", "apply")
-}
-
 func TestRunCommentCommand_DisableDisableAutoplan(t *testing.T) {
 	t.Log("if \"DisableAutoplan is true\" are disabled and we are silencing return and do not comment with error")
 	setup(t)

server/events/db/boltdb.go

@@ -17,15 +17,17 @@ import (
 
 // BoltDB is a database using BoltDB
 type BoltDB struct {
-	db              *bolt.DB
-	locksBucketName []byte
-	pullsBucketName []byte
+	db                    *bolt.DB
+	locksBucketName       []byte
+	pullsBucketName       []byte
+	globalLocksBucketName []byte
 }
 
 const (
-	locksBucketName  = "runLocks"
-	pullsBucketName  = "pulls"
-	pullKeySeparator = "::"
+	locksBucketName       = "runLocks"
+	pullsBucketName       = "pulls"
+	globalLocksBucketName = "globalLocks"
+	pullKeySeparator      = "::"
 )
 
 // New returns a valid locker. We need to be able to write to dataDir
@@ -50,18 +52,31 @@ func New(dataDir string) (*BoltDB, error) {
 		if _, err = tx.CreateBucketIfNotExists([]byte(pullsBucketName)); err != nil {
 			return errors.Wrapf(err, "creating bucket %q", pullsBucketName)
 		}
+		if _, err = tx.CreateBucketIfNotExists([]byte(globalLocksBucketName)); err != nil {
+			return errors.Wrapf(err, "creating bucket %q", globalLocksBucketName)
+		}
 		return nil
 	})
 	if err != nil {
 		return nil, errors.Wrap(err, "starting BoltDB")
 	}
 	// todo: close BoltDB when server is sigtermed
-	return &BoltDB{db: db, locksBucketName: []byte(locksBucketName), pullsBucketName: []byte(pullsBucketName)}, nil
+	return &BoltDB{
+		db:                    db,
+		locksBucketName:       []byte(locksBucketName),
+		pullsBucketName:       []byte(pullsBucketName),
+		globalLocksBucketName: []byte(globalLocksBucketName),
+	}, nil
 }
 
 // NewWithDB is used for testing.
-func NewWithDB(db *bolt.DB, bucket string) (*BoltDB, error) {
-	return &BoltDB{db: db, locksBucketName: []byte(bucket), pullsBucketName: []byte(pullsBucketName)}, nil
+func NewWithDB(db *bolt.DB, bucket string, globalBucket string) (*BoltDB, error) {
+	return &BoltDB{
+		db:                    db,
+		locksBucketName:       []byte(bucket),
+		pullsBucketName:       []byte(pullsBucketName),
+		globalLocksBucketName: []byte(globalBucket),
+	}, nil
 }
 
 // TryLock attempts to create a new lock. If the lock is
@@ -155,6 +170,87 @@ func (b *BoltDB) List() ([]models.ProjectLock, error) {
 	return locks, nil
 }
 
+// LockCommand attempts to create a new lock for a CommandName.
+// If the lock doesn't exists, it will create a lock and return a pointer to it.
+// If the lock already exists, it will return an "lock already exists" error
+func (b *BoltDB) LockCommand(cmdName models.CommandName, lockTime time.Time) (*models.CommandLock, error) {
+	lock := models.CommandLock{
+		CommandName: cmdName,
+		LockMetadata: models.LockMetadata{
+			UnixTime: lockTime.Unix(),
+		},
+	}
+
+	newLockSerialized, _ := json.Marshal(lock)
+	transactionErr := b.db.Update(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket(b.globalLocksBucketName)
+
+		currLockSerialized := bucket.Get([]byte(b.commandLockKey(cmdName)))
+		if currLockSerialized != nil {
+			return errors.New("lock already exists")
+		}
+
+		// This will only error on readonly buckets, it's okay to ignore.
+		bucket.Put([]byte(b.commandLockKey(cmdName)), newLockSerialized) // nolint: errcheck
+		return nil
+	})
+
+	if transactionErr != nil {
+		return nil, errors.Wrap(transactionErr, "db transaction failed")
+	}
+
+	return &lock, nil
+}
+
+// UnlockCommand removes CommandName lock if present.
+// If there are no lock it returns an error.
+func (b *BoltDB) UnlockCommand(cmdName models.CommandName) error {
+	transactionErr := b.db.Update(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket(b.globalLocksBucketName)
+
+		if l := bucket.Get([]byte(b.commandLockKey(cmdName))); l == nil {
+			return errors.New("no lock exists")
+		}
+
+		return bucket.Delete([]byte(b.commandLockKey(cmdName)))
+	})
+
+	if transactionErr != nil {
+		return errors.Wrap(transactionErr, "db transaction failed")
+	}
+
+	return nil
+}
+
+// CheckCommandLock checks if CommandName lock was set.
+// If the lock exists return the pointer to the lock object, otherwise return nil
+func (b *BoltDB) CheckCommandLock(cmdName models.CommandName) (*models.CommandLock, error) {
+	cmdLock := models.CommandLock{}
+
+	found := false
+
+	err := b.db.View(func(tx *bolt.Tx) error {
+		bucket := tx.Bucket(b.globalLocksBucketName)
+
+		serializedLock := bucket.Get([]byte(b.commandLockKey(cmdName)))
+
+		if serializedLock != nil {
+			if err := json.Unmarshal(serializedLock, &cmdLock); err != nil {
+				return errors.Wrap(err, "failed to deserialize UserConfig")
+			}
+			found = true
+		}
+
+		return nil
+	})
+
+	if found {
+		return &cmdLock, err
+	}
+
+	return nil, err
+}
+
 // UnlockByPull deletes all locks associated with that pull request and returns them.
 func (b *BoltDB) UnlockByPull(repoFullName string, pullNum int) ([]models.ProjectLock, error) {
 	var locks []models.ProjectLock
@@ -355,6 +451,10 @@ func (b *BoltDB) pullKey(pull models.PullRequest) ([]byte, error) {
 		nil
 }
 
+func (b *BoltDB) commandLockKey(cmdName models.CommandName) string {
+	return fmt.Sprintf("%s/lock", cmdName)
+}
+
 func (b *BoltDB) lockKey(p models.Project, workspace string) string {
 	return fmt.Sprintf("%s/%s/%s", p.RepoFullName, p.Path, workspace)
 }