Merge pull request #226 from luxas/manifest_dir

Rename GitStorage into ManifestStorage

pkg/storage/gitops/gitdir.go → pkg/gitops/gitdir/gitdir.go

@@ -1,4 +1,4 @@
-package gitops
+package gitdir
 
 import (
 	"context"

pkg/gitops/gitops.go

@@ -11,35 +11,53 @@ import (
 	meta "github.com/weaveworks/ignite/pkg/apis/meta/v1alpha1"
 	"github.com/weaveworks/ignite/pkg/client"
 	"github.com/weaveworks/ignite/pkg/dmlegacy"
+	"github.com/weaveworks/ignite/pkg/gitops/gitdir"
 	"github.com/weaveworks/ignite/pkg/operations"
 	"github.com/weaveworks/ignite/pkg/storage"
-	"github.com/weaveworks/ignite/pkg/storage/gitops"
+	"github.com/weaveworks/ignite/pkg/storage/manifest"
 	"github.com/weaveworks/ignite/pkg/util"
 )
 
 var (
-	vmMap map[meta.UID]*api.VM
-	c     *client.Client
+	vmMap           map[meta.UID]*api.VM
+	c               *client.Client
+	gitDir          *gitdir.GitDirectory
+	syncInterval, _ = time.ParseDuration("10s")
 )
 
+const dataDir = "/tmp/ignite-gitops"
+
 func RunLoop(url, branch string) error {
 	log.Printf("Starting GitOps loop for repo at %q\n", url)
 	log.Printf("Whenever changes are pushed to the %s branch, Ignite will apply the desired state locally\n", branch)
 	log.Println("Initializing the Git repo...")
 
-	s := gitops.NewGitOpsStorage(url, branch)
-	// Wrap the GitOps storage with a cache for better performance
+	// Construct a manifest storage for the path backed by git
+	s := manifest.NewManifestStorage(dataDir)
+	// Wrap the Manifest Storage with a cache for better performance, and create a client
 	c = client.NewClient(storage.NewCache(s))
+	// Construct the GitDirectory implementation which backs the storage
+	gitDir = gitdir.NewGitDirectory(url, dataDir, branch, syncInterval)
+	// Start the GitDirectory sync loop
+	gitDir.StartLoop()
 
 	for {
-		if !s.Ready() {
+		if !gitDir.Ready() {
 			// poll until the git repo is initialized
 			time.Sleep(5 * time.Second)
 			continue
 		}
 
+		// Wait for changes to happen in the Git repo
 		log.Println("Waiting for updates in the Git repo...")
-		diff := s.WaitForUpdate()
+		_ = gitDir.WaitForUpdate()
+
+		// When we know the underlying state has changed, reload the storage mappings, and get what's changed
+		diff, err := s.Sync()
+		if err != nil {
+			log.Warnf("Syncing the new directory state returned an error: %v. Retrying...", err)
+			continue
+		}
 
 		list, err := c.VMs().List()
 		if err != nil {
@@ -53,7 +71,7 @@ func RunLoop(url, branch string) error {
 		for _, file := range diff {
 			vm := vmMap[file.APIType.UID]
 			if vm == nil {
-				if file.Type != gitops.UpdateTypeDeleted {
+				if file.Type != manifest.UpdateTypeDeleted {
 					// This is unexpected
 					log.Warn("Skipping %s of %s with UID %s, no such object found through the client.", file.Type, file.APIType.GetKind(), file.APIType.GetUID())
 					continue
@@ -84,17 +102,17 @@ func RunLoop(url, branch string) error {
 
 			// TODO: At the moment there aren't running in parallel, shall they?
 			switch file.Type {
-			case gitops.UpdateTypeCreated:
+			case manifest.UpdateTypeCreated:
 				// TODO: Run this as a goroutine
 				runHandle(wg, func() error {
 					return handleCreate(vm)
 				})
-			case gitops.UpdateTypeChanged:
+			case manifest.UpdateTypeChanged:
 				// TODO: Run this as a goroutine
 				runHandle(wg, func() error {
 					return handleChange(vm)
 				})
-			case gitops.UpdateTypeDeleted:
+			case manifest.UpdateTypeDeleted:
 				// TODO: Run this as a goroutine
 				runHandle(wg, func() error {
 					// TODO: Temporary VM Object for removal

pkg/storage/gitops/rawstorage.go → pkg/storage/manifest/rawstorage.go

@@ -1,4 +1,4 @@
-package gitops
+package manifest
 
 import (
 	"crypto/sha256"
@@ -19,9 +19,9 @@ import (
 
 var splitDirsRegex = regexp.MustCompile(`/?([a-z0-9]+)(/[a-z0-9]*)*`)
 
-func NewGitRawStorage(gitDir, underlyingDir string) *GitRawStorage {
-	return &GitRawStorage{
-		gitDir: gitDir,
+func NewManifestRawStorage(manifestDir, underlyingDir string) *ManifestRawStorage {
+	return &ManifestRawStorage{
+		manifestDir: manifestDir,
 		gitPathPrefixes: map[string]bool{ // we only check in VM state into git atm
 			"/vm": true, // TODO: construct this in a better way
 		},
@@ -46,9 +46,9 @@ type UpdatedFile struct {
 	APIType  *meta.APIType
 }
 
-type GitRawStorage struct {
+type ManifestRawStorage struct {
 	// directory that is managed by git
-	gitDir string
+	manifestDir string
 	// keyFileMap maps the virtual key path to real file paths in the repo
 	keyFileMap map[string]*UpdatedFile
 	// byKind maps a kind to many virtual key paths for the storage impl
@@ -59,14 +59,14 @@ type GitRawStorage struct {
 	passthrough storage.RawStorage
 }
 
-func (r *GitRawStorage) Sync() (UpdatedFiles, error) {
+func (r *ManifestRawStorage) Sync() (UpdatedFiles, error) {
 	// provide empty placeholders for new data, overwrite .keyFileMap and .byKind in the end
 	newKeyFileMap := map[string]*UpdatedFile{}
 	newByKind := map[string][]string{}
 	// a slice of files that
 	diff := UpdatedFiles{}
-	// walk the git repo
-	dirToWalk := r.gitDir
+	// walk the manifest dir
+	dirToWalk := r.manifestDir
 	if !strings.HasSuffix(dirToWalk, "/") {
 		// filepath.Walk needs a trailing slash to start traversing the directory
 		dirToWalk += "/"
@@ -188,35 +188,35 @@ func sha256sum(content []byte) string {
 	return fmt.Sprintf("%x", hasher.Sum(nil))
 }
 
-func (r *GitRawStorage) gitRelativePath(fullPath string) string {
-	return strings.TrimPrefix(fullPath, r.gitDir+"/")
+func (r *ManifestRawStorage) gitRelativePath(fullPath string) string {
+	return strings.TrimPrefix(fullPath, r.manifestDir+"/")
 }
 
-func (r *GitRawStorage) realPath(key string) string {
+func (r *ManifestRawStorage) realPath(key string) string {
 	// The "/" prefix is enforced
 	if !strings.HasPrefix(key, "/") {
 		key = "/" + key
 	}
 
 	info, ok := r.keyFileMap[key]
 	if !ok {
-		log.Debugf("GitRawStorage.realPath returned an empty string for key %s", key)
+		log.Debugf("ManifestRawStorage.realPath returned an empty string for key %s", key)
 		return ""
 	}
 
 	return info.GitPath
 }
 
-func (r *GitRawStorage) shouldPassthrough(key string) bool {
+func (r *ManifestRawStorage) shouldPassthrough(key string) bool {
 	// firstDirName is e.g. "vm" when key is "/vm/foobar123"
 	firstDirName := splitDirsRegex.FindStringSubmatch(key)[1]
 	// check if this kind should be managed by git. if it's git-managed return false
 	_, ok := r.gitPathPrefixes[fmt.Sprintf("/%s", strings.ToLower(firstDirName))]
 	return !ok
 }
 
-func (r *GitRawStorage) Read(key string) ([]byte, error) {
-	log.Debugf("GitRawStorage.Read: %q", key)
+func (r *ManifestRawStorage) Read(key string) ([]byte, error) {
+	log.Debugf("ManifestRawStorage.Read: %q", key)
 	if r.shouldPassthrough(key) {
 		return r.passthrough.Read(key)
 	}
@@ -225,8 +225,8 @@ func (r *GitRawStorage) Read(key string) ([]byte, error) {
 	return ioutil.ReadFile(file)
 }
 
-func (r *GitRawStorage) Exists(key string) bool {
-	log.Debugf("GitRawStorage.Exists: %q", key)
+func (r *ManifestRawStorage) Exists(key string) bool {
+	log.Debugf("ManifestRawStorage.Exists: %q", key)
 	if r.shouldPassthrough(key) {
 		return r.passthrough.Exists(key)
 	}
@@ -235,8 +235,8 @@ func (r *GitRawStorage) Exists(key string) bool {
 	return util.FileExists(file)
 }
 
-func (r *GitRawStorage) Write(key string, content []byte) error {
-	log.Debugf("GitRawStorage.Write: %q", key)
+func (r *ManifestRawStorage) Write(key string, content []byte) error {
+	log.Debugf("ManifestRawStorage.Write: %q", key)
 	// Write always writes to the underlying (expected) place, and to Git
 	if err := r.passthrough.Write(key, content); err != nil {
 		return err
@@ -257,8 +257,8 @@ func (r *GitRawStorage) Write(key string, content []byte) error {
 	return nil
 }
 
-func (r *GitRawStorage) Delete(key string) error {
-	log.Debugf("GitRawStorage.Delete: %q", key)
+func (r *ManifestRawStorage) Delete(key string) error {
+	log.Debugf("ManifestRawStorage.Delete: %q", key)
 	// Delete always deletes in the underlying (expected) place, and in Git
 	if err := r.passthrough.Delete(key); err != nil {
 		return err
@@ -279,8 +279,8 @@ func (r *GitRawStorage) Delete(key string) error {
 	// TODO: Do a git commit here!
 }
 
-func (r *GitRawStorage) List(parentKey string) ([]string, error) {
-	log.Debugf("GitRawStorage.List: %q", parentKey)
+func (r *ManifestRawStorage) List(parentKey string) ([]string, error) {
+	log.Debugf("ManifestRawStorage.List: %q", parentKey)
 	if r.shouldPassthrough(parentKey) {
 		return r.passthrough.List(parentKey)
 	}

pkg/storage/manifest/storage.go

@@ -0,0 +1,25 @@
+package manifest
+
+import (
+	"github.com/weaveworks/ignite/pkg/apis/ignite/scheme"
+	"github.com/weaveworks/ignite/pkg/constants"
+	"github.com/weaveworks/ignite/pkg/storage"
+)
+
+func NewManifestStorage(dataDir string) *ManifestStorage {
+	gitRaw := NewManifestRawStorage(dataDir, constants.DATA_DIR)
+	return &ManifestStorage{
+		gitRaw:  gitRaw,
+		Storage: storage.NewGenericStorage(gitRaw, scheme.Serializer),
+	}
+}
+
+// ManifestStorage implements the storage interface for GitOps purposes
+type ManifestStorage struct {
+	storage.Storage
+	gitRaw *ManifestRawStorage
+}
+
+func (s *ManifestStorage) Sync() (UpdatedFiles, error) {
+	return s.gitRaw.Sync()
+}