Skip to content
This repository has been archived by the owner on Dec 7, 2023. It is now read-only.

Rename GitStorage into ManifestStorage #226

Merged
merged 1 commit into from
Jul 19, 2019
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package gitops
package gitdir

import (
"context"
Expand Down
40 changes: 29 additions & 11 deletions pkg/gitops/gitops.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,35 +11,53 @@ import (
meta "github.com/weaveworks/ignite/pkg/apis/meta/v1alpha1"
"github.com/weaveworks/ignite/pkg/client"
"github.com/weaveworks/ignite/pkg/dmlegacy"
"github.com/weaveworks/ignite/pkg/gitops/gitdir"
"github.com/weaveworks/ignite/pkg/operations"
"github.com/weaveworks/ignite/pkg/storage"
"github.com/weaveworks/ignite/pkg/storage/gitops"
"github.com/weaveworks/ignite/pkg/storage/manifest"
"github.com/weaveworks/ignite/pkg/util"
)

var (
vmMap map[meta.UID]*api.VM
c *client.Client
vmMap map[meta.UID]*api.VM
c *client.Client
gitDir *gitdir.GitDirectory
syncInterval, _ = time.ParseDuration("10s")
)

const dataDir = "/tmp/ignite-gitops"

func RunLoop(url, branch string) error {
log.Printf("Starting GitOps loop for repo at %q\n", url)
log.Printf("Whenever changes are pushed to the %s branch, Ignite will apply the desired state locally\n", branch)
log.Println("Initializing the Git repo...")

s := gitops.NewGitOpsStorage(url, branch)
// Wrap the GitOps storage with a cache for better performance
// Construct a manifest storage for the path backed by git
s := manifest.NewManifestStorage(dataDir)
// Wrap the Manifest Storage with a cache for better performance, and create a client
c = client.NewClient(storage.NewCache(s))
// Construct the GitDirectory implementation which backs the storage
gitDir = gitdir.NewGitDirectory(url, dataDir, branch, syncInterval)
// Start the GitDirectory sync loop
gitDir.StartLoop()

for {
if !s.Ready() {
if !gitDir.Ready() {
// poll until the git repo is initialized
time.Sleep(5 * time.Second)
continue
}

// Wait for changes to happen in the Git repo
log.Println("Waiting for updates in the Git repo...")
diff := s.WaitForUpdate()
_ = gitDir.WaitForUpdate()

// When we know the underlying state has changed, reload the storage mappings, and get what's changed
diff, err := s.Sync()
if err != nil {
log.Warnf("Syncing the new directory state returned an error: %v. Retrying...", err)
continue
}

list, err := c.VMs().List()
if err != nil {
Expand All @@ -53,7 +71,7 @@ func RunLoop(url, branch string) error {
for _, file := range diff {
vm := vmMap[file.APIType.UID]
if vm == nil {
if file.Type != gitops.UpdateTypeDeleted {
if file.Type != manifest.UpdateTypeDeleted {
// This is unexpected
log.Warn("Skipping %s of %s with UID %s, no such object found through the client.", file.Type, file.APIType.GetKind(), file.APIType.GetUID())
continue
Expand Down Expand Up @@ -84,17 +102,17 @@ func RunLoop(url, branch string) error {

// TODO: At the moment there aren't running in parallel, shall they?
switch file.Type {
case gitops.UpdateTypeCreated:
case manifest.UpdateTypeCreated:
// TODO: Run this as a goroutine
runHandle(wg, func() error {
return handleCreate(vm)
})
case gitops.UpdateTypeChanged:
case manifest.UpdateTypeChanged:
// TODO: Run this as a goroutine
runHandle(wg, func() error {
return handleChange(vm)
})
case gitops.UpdateTypeDeleted:
case manifest.UpdateTypeDeleted:
// TODO: Run this as a goroutine
runHandle(wg, func() error {
// TODO: Temporary VM Object for removal
Expand Down
60 changes: 0 additions & 60 deletions pkg/storage/gitops/storage.go

This file was deleted.

Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package gitops
package manifest

import (
"crypto/sha256"
Expand All @@ -19,9 +19,9 @@ import (

var splitDirsRegex = regexp.MustCompile(`/?([a-z0-9]+)(/[a-z0-9]*)*`)

func NewGitRawStorage(gitDir, underlyingDir string) *GitRawStorage {
return &GitRawStorage{
gitDir: gitDir,
func NewManifestRawStorage(manifestDir, underlyingDir string) *ManifestRawStorage {
return &ManifestRawStorage{
manifestDir: manifestDir,
gitPathPrefixes: map[string]bool{ // we only check in VM state into git atm
"/vm": true, // TODO: construct this in a better way
},
Expand All @@ -46,9 +46,9 @@ type UpdatedFile struct {
APIType *meta.APIType
}

type GitRawStorage struct {
type ManifestRawStorage struct {
// directory that is managed by git
gitDir string
manifestDir string
// keyFileMap maps the virtual key path to real file paths in the repo
keyFileMap map[string]*UpdatedFile
// byKind maps a kind to many virtual key paths for the storage impl
Expand All @@ -59,14 +59,14 @@ type GitRawStorage struct {
passthrough storage.RawStorage
}

func (r *GitRawStorage) Sync() (UpdatedFiles, error) {
func (r *ManifestRawStorage) Sync() (UpdatedFiles, error) {
// provide empty placeholders for new data, overwrite .keyFileMap and .byKind in the end
newKeyFileMap := map[string]*UpdatedFile{}
newByKind := map[string][]string{}
// a slice of files that
diff := UpdatedFiles{}
// walk the git repo
dirToWalk := r.gitDir
// walk the manifest dir
dirToWalk := r.manifestDir
if !strings.HasSuffix(dirToWalk, "/") {
// filepath.Walk needs a trailing slash to start traversing the directory
dirToWalk += "/"
Expand Down Expand Up @@ -188,35 +188,35 @@ func sha256sum(content []byte) string {
return fmt.Sprintf("%x", hasher.Sum(nil))
}

func (r *GitRawStorage) gitRelativePath(fullPath string) string {
return strings.TrimPrefix(fullPath, r.gitDir+"/")
func (r *ManifestRawStorage) gitRelativePath(fullPath string) string {
return strings.TrimPrefix(fullPath, r.manifestDir+"/")
}

func (r *GitRawStorage) realPath(key string) string {
func (r *ManifestRawStorage) realPath(key string) string {
// The "/" prefix is enforced
if !strings.HasPrefix(key, "/") {
key = "/" + key
}

info, ok := r.keyFileMap[key]
if !ok {
log.Debugf("GitRawStorage.realPath returned an empty string for key %s", key)
log.Debugf("ManifestRawStorage.realPath returned an empty string for key %s", key)
return ""
}

return info.GitPath
}

func (r *GitRawStorage) shouldPassthrough(key string) bool {
func (r *ManifestRawStorage) shouldPassthrough(key string) bool {
// firstDirName is e.g. "vm" when key is "/vm/foobar123"
firstDirName := splitDirsRegex.FindStringSubmatch(key)[1]
// check if this kind should be managed by git. if it's git-managed return false
_, ok := r.gitPathPrefixes[fmt.Sprintf("/%s", strings.ToLower(firstDirName))]
return !ok
}

func (r *GitRawStorage) Read(key string) ([]byte, error) {
log.Debugf("GitRawStorage.Read: %q", key)
func (r *ManifestRawStorage) Read(key string) ([]byte, error) {
log.Debugf("ManifestRawStorage.Read: %q", key)
if r.shouldPassthrough(key) {
return r.passthrough.Read(key)
}
Expand All @@ -225,8 +225,8 @@ func (r *GitRawStorage) Read(key string) ([]byte, error) {
return ioutil.ReadFile(file)
}

func (r *GitRawStorage) Exists(key string) bool {
log.Debugf("GitRawStorage.Exists: %q", key)
func (r *ManifestRawStorage) Exists(key string) bool {
log.Debugf("ManifestRawStorage.Exists: %q", key)
if r.shouldPassthrough(key) {
return r.passthrough.Exists(key)
}
Expand All @@ -235,8 +235,8 @@ func (r *GitRawStorage) Exists(key string) bool {
return util.FileExists(file)
}

func (r *GitRawStorage) Write(key string, content []byte) error {
log.Debugf("GitRawStorage.Write: %q", key)
func (r *ManifestRawStorage) Write(key string, content []byte) error {
log.Debugf("ManifestRawStorage.Write: %q", key)
// Write always writes to the underlying (expected) place, and to Git
if err := r.passthrough.Write(key, content); err != nil {
return err
Expand All @@ -257,8 +257,8 @@ func (r *GitRawStorage) Write(key string, content []byte) error {
return nil
}

func (r *GitRawStorage) Delete(key string) error {
log.Debugf("GitRawStorage.Delete: %q", key)
func (r *ManifestRawStorage) Delete(key string) error {
log.Debugf("ManifestRawStorage.Delete: %q", key)
// Delete always deletes in the underlying (expected) place, and in Git
if err := r.passthrough.Delete(key); err != nil {
return err
Expand All @@ -279,8 +279,8 @@ func (r *GitRawStorage) Delete(key string) error {
// TODO: Do a git commit here!
}

func (r *GitRawStorage) List(parentKey string) ([]string, error) {
log.Debugf("GitRawStorage.List: %q", parentKey)
func (r *ManifestRawStorage) List(parentKey string) ([]string, error) {
log.Debugf("ManifestRawStorage.List: %q", parentKey)
if r.shouldPassthrough(parentKey) {
return r.passthrough.List(parentKey)
}
Expand Down
25 changes: 25 additions & 0 deletions pkg/storage/manifest/storage.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
package manifest

import (
"github.com/weaveworks/ignite/pkg/apis/ignite/scheme"
"github.com/weaveworks/ignite/pkg/constants"
"github.com/weaveworks/ignite/pkg/storage"
)

func NewManifestStorage(dataDir string) *ManifestStorage {
gitRaw := NewManifestRawStorage(dataDir, constants.DATA_DIR)
return &ManifestStorage{
gitRaw: gitRaw,
Storage: storage.NewGenericStorage(gitRaw, scheme.Serializer),
}
}

// ManifestStorage implements the storage interface for GitOps purposes
type ManifestStorage struct {
storage.Storage
gitRaw *ManifestRawStorage
}

func (s *ManifestStorage) Sync() (UpdatedFiles, error) {
return s.gitRaw.Sync()
}