From 336cfe07fae9761f16dc66113773f16c9ef673ff Mon Sep 17 00:00:00 2001 From: Tonis Tiigi <tonistiigi@gmail.com> Date: Mon, 3 Jul 2017 16:08:20 -0700 Subject: [PATCH 1/3] persistent metadata helpers for snapshots Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com> --- cache/manager.go | 112 +++++++------ cache/metadata.go | 40 +++++ cache/metadata/metadata.go | 284 ++++++++++++++++++++++++++++++++ cache/metadata/metadata_test.go | 166 +++++++++++++++++++ cache/refs.go | 50 +++--- 5 files changed, 586 insertions(+), 66 deletions(-) create mode 100644 cache/metadata.go create mode 100644 cache/metadata/metadata.go create mode 100644 cache/metadata/metadata_test.go diff --git a/cache/manager.go b/cache/manager.go index 5bb1b8185c4e..3a3f68679f8d 100644 --- a/cache/manager.go +++ b/cache/manager.go @@ -6,8 +6,9 @@ import ( "path/filepath" "sync" - "github.com/boltdb/bolt" + "github.com/Sirupsen/logrus" cdsnapshot "github.com/containerd/containerd/snapshot" + "github.com/moby/buildkit/cache/metadata" "github.com/moby/buildkit/client" "github.com/moby/buildkit/identity" "github.com/moby/buildkit/snapshot" @@ -48,10 +49,10 @@ type Manager interface { } type cacheManager struct { - db *bolt.DB // note: no particual reason for bolt records map[string]*cacheRecord mu sync.Mutex ManagerOpt + md *metadata.Store } func NewManager(opt ManagerOpt) (Manager, error) { @@ -59,19 +60,18 @@ func NewManager(opt ManagerOpt) (Manager, error) { return nil, errors.Wrapf(err, "failed to create %s", opt.Root) } - p := filepath.Join(opt.Root, dbFile) - db, err := bolt.Open(p, 0600, nil) + md, err := metadata.NewStore(filepath.Join(opt.Root, dbFile)) if err != nil { - return nil, errors.Wrapf(err, "failed to open database file %s", p) + return nil, err } cm := &cacheManager{ ManagerOpt: opt, - db: db, + md: md, records: make(map[string]*cacheRecord), } - if err := cm.init(); err != nil { + if err := cm.init(context.TODO()); err != nil { return nil, err } @@ -80,17 +80,24 @@ func NewManager(opt ManagerOpt) (Manager, error) { return cm, nil } -func (cm *cacheManager) init() error { - // load all refs from db - // compare with the walk from Snapshotter - // delete items that are not in db (or implement broken transaction detection) - // keep all refs in memory(maybe in future work on disk only or with lru) +func (cm *cacheManager) init(ctx context.Context) error { + items, err := cm.md.All() + if err != nil { + return err + } + + for _, si := range items { + if _, err := cm.load(ctx, si.ID()); err != nil { + logrus.Debugf("could not load snapshot %s, %v", si.ID(), err) + cm.md.Clear(si.ID()) + } + } return nil } func (cm *cacheManager) Close() error { // TODO: allocate internal context and cancel it here - return cm.db.Close() + return cm.md.Close() } func (cm *cacheManager) Get(ctx context.Context, id string) (ImmutableRef, error) { @@ -99,37 +106,17 @@ func (cm *cacheManager) Get(ctx context.Context, id string) (ImmutableRef, error return cm.get(ctx, id) } func (cm *cacheManager) get(ctx context.Context, id string) (ImmutableRef, error) { - rec, ok := cm.records[id] - if !ok { - info, err := cm.Snapshotter.Stat(ctx, id) - if err != nil { - return nil, err - } - if info.Kind != cdsnapshot.KindCommitted { - return nil, errors.Wrapf(errInvalid, "can't lazy load active %s", id) - } - - var parent ImmutableRef - if info.Parent != "" { - parent, err = cm.get(ctx, info.Parent) - if err != nil { - return nil, err - } - } - - rec = &cacheRecord{ - id: id, - cm: cm, - refs: make(map[Mountable]struct{}), - parent: parent, - size: sizeUnknown, - } - cm.records[id] = rec // TODO: store to db + rec, err := cm.load(ctx, id) + if err != nil { + return nil, err } - rec.mu.Lock() defer rec.mu.Unlock() + if rec.mutable { + return nil, errors.Wrapf(errInvalid, "invalid mutable ref %s", id) + } + if rec.mutable && !rec.frozen { if len(rec.refs) != 0 { return nil, errors.Wrapf(errLocked, "%s is locked", id) @@ -140,6 +127,38 @@ func (cm *cacheManager) get(ctx context.Context, id string) (ImmutableRef, error return rec.ref(), nil } + +func (cm *cacheManager) load(ctx context.Context, id string) (*cacheRecord, error) { + if rec, ok := cm.records[id]; ok { + return rec, nil + } + + info, err := cm.Snapshotter.Stat(ctx, id) + if err != nil { + return nil, errors.Wrap(errNotFound, err.Error()) + } + + var parent ImmutableRef + if info.Parent != "" { + parent, err = cm.get(ctx, info.Parent) + if err != nil { + return nil, err + } + } + + md, _ := cm.md.Get(id) + + rec := &cacheRecord{ + mutable: info.Kind != cdsnapshot.KindCommitted, + cm: cm, + refs: make(map[Mountable]struct{}), + parent: parent, + md: &md, + } + cm.records[id] = rec // TODO: store to db + return rec, nil +} + func (cm *cacheManager) New(ctx context.Context, s ImmutableRef) (MutableRef, error) { id := identity.NewID() @@ -161,13 +180,14 @@ func (cm *cacheManager) New(ctx context.Context, s ImmutableRef) (MutableRef, er return nil, errors.Wrapf(err, "failed to prepare %s", id) } + md, _ := cm.md.Get(id) + rec := &cacheRecord{ mutable: true, - id: id, cm: cm, refs: make(map[Mountable]struct{}), parent: parent, - size: sizeUnknown, + md: &md, } cm.mu.Lock() @@ -181,9 +201,9 @@ func (cm *cacheManager) GetMutable(ctx context.Context, id string) (MutableRef, cm.mu.Lock() defer cm.mu.Unlock() - rec, ok := cm.records[id] - if !ok { - return nil, errors.Wrapf(errNotFound, "%s not found", id) + rec, err := cm.load(ctx, id) + if err != nil { + return nil, err } rec.mu.Lock() @@ -217,7 +237,7 @@ func (cm *cacheManager) DiskUsage(ctx context.Context) ([]*client.UsageInfo, err c := &cacheUsageInfo{ refs: len(cr.refs), mutable: cr.mutable, - size: cr.size, + size: getSize(cr.md), } if cr.parent != nil { c.parent = cr.parent.ID() diff --git a/cache/metadata.go b/cache/metadata.go new file mode 100644 index 000000000000..5691daaa9cc1 --- /dev/null +++ b/cache/metadata.go @@ -0,0 +1,40 @@ +package cache + +import ( + "github.com/boltdb/bolt" + "github.com/moby/buildkit/cache/metadata" + "github.com/pkg/errors" +) + +// Fields to be added: +// Size int64 +// AccessTime int64 +// Tags +// Descr +// CachePolicy + +const sizeUnknown int64 = -1 +const keySize = "snapshot.size" + +func setSize(si *metadata.StorageItem, s int64) error { + v, err := metadata.NewValue(s) + if err != nil { + return errors.Wrap(err, "failed to create size value") + } + si.Queue(func(b *bolt.Bucket) error { + return si.SetValue(b, keySize, *v) + }) + return nil +} + +func getSize(si *metadata.StorageItem) int64 { + v := si.Get(keySize) + if v == nil { + return sizeUnknown + } + var size int64 + if err := v.Unmarshal(&size); err != nil { + return sizeUnknown + } + return size +} diff --git a/cache/metadata/metadata.go b/cache/metadata/metadata.go new file mode 100644 index 000000000000..6a24d83fad29 --- /dev/null +++ b/cache/metadata/metadata.go @@ -0,0 +1,284 @@ +package metadata + +import ( + "encoding/json" + "strings" + + "github.com/Sirupsen/logrus" + "github.com/boltdb/bolt" + "github.com/pkg/errors" +) + +const ( + mainBucket = "_main" + indexBucket = "_index" +) + +var errNotFound = errors.Errorf("not found") + +type Store struct { + db *bolt.DB +} + +func NewStore(dbPath string) (*Store, error) { + db, err := bolt.Open(dbPath, 0600, nil) + if err != nil { + return nil, errors.Wrapf(err, "failed to open database file %s", dbPath) + } + return &Store{db: db}, nil +} + +func (s *Store) All() ([]StorageItem, error) { + var out []StorageItem + err := s.db.View(func(tx *bolt.Tx) error { + b := tx.Bucket([]byte(mainBucket)) + if b == nil { + return nil + } + return b.ForEach(func(key, _ []byte) error { + b := b.Bucket(key) + if b == nil { + return nil + } + si, err := newStorageItem(string(key), b, s) + if err != nil { + return err + } + out = append(out, si) + return nil + }) + }) + return out, err +} + +func (s *Store) Search(index string) ([]StorageItem, error) { + var out []StorageItem + err := s.db.View(func(tx *bolt.Tx) error { + b := tx.Bucket([]byte(indexBucket)) + if b == nil { + return nil + } + main := tx.Bucket([]byte(mainBucket)) + if main == nil { + return nil + } + index = indexKey(index, "") + c := b.Cursor() + k, _ := c.Seek([]byte(index)) + for { + if k != nil && strings.HasPrefix(string(k), index) { + itemID := strings.TrimPrefix(string(k), index) + k, _ = c.Next() + b := main.Bucket([]byte(itemID)) + if b == nil { + logrus.Errorf("index pointing to missing record %s", itemID) + continue + } + si, err := newStorageItem(itemID, b, s) + if err != nil { + return err + } + out = append(out, si) + } else { + break + } + } + return nil + }) + return out, err +} + +func (s *Store) View(id string, fn func(b *bolt.Bucket) error) error { + return s.db.View(func(tx *bolt.Tx) error { + b := tx.Bucket([]byte(mainBucket)) + if b == nil { + return errors.WithStack(errNotFound) + } + b = b.Bucket([]byte(id)) + if b == nil { + return errors.WithStack(errNotFound) + } + return fn(b) + }) +} + +func (s *Store) Clear(id string) error { + return s.db.Update(func(tx *bolt.Tx) error { + main := tx.Bucket([]byte(mainBucket)) + if main == nil { + return nil + } + b := main.Bucket([]byte(id)) + if b == nil { + return nil + } + si, err := newStorageItem(id, b, s) + if err != nil { + return err + } + if indexes := si.Indexes(); len(indexes) > 0 { + b := tx.Bucket([]byte(indexBucket)) + if b != nil { + for _, index := range indexes { + if err := b.Delete([]byte(indexKey(index, id))); err != nil { + return err + } + } + } + } + return main.DeleteBucket([]byte(id)) + }) +} + +func (s *Store) Update(id string, fn func(b *bolt.Bucket) error) error { + return s.db.Update(func(tx *bolt.Tx) error { + b, err := tx.CreateBucketIfNotExists([]byte(mainBucket)) + if err != nil { + return err + } + b, err = b.CreateBucketIfNotExists([]byte(id)) + if err != nil { + return err + } + return fn(b) + }) +} + +func (s *Store) Get(id string) (StorageItem, bool) { + empty := func() StorageItem { + si, _ := newStorageItem(id, nil, s) + return si + } + tx, err := s.db.Begin(false) + if err != nil { + return empty(), false + } + defer tx.Rollback() + b := tx.Bucket([]byte(mainBucket)) + if b == nil { + return empty(), false + } + b = b.Bucket([]byte(id)) + if b == nil { + return empty(), false + } + si, _ := newStorageItem(id, b, s) + return si, true +} + +func (s *Store) Close() error { + return s.db.Close() +} + +type StorageItem struct { + id string + values map[string]*Value + queue []func(*bolt.Bucket) error + storage *Store +} + +func newStorageItem(id string, b *bolt.Bucket, s *Store) (StorageItem, error) { + si := StorageItem{ + id: id, + storage: s, + values: make(map[string]*Value), + } + if b != nil { + if err := b.ForEach(func(k, v []byte) error { + var sv Value + if err := json.Unmarshal(v, &sv); err != nil { + return err + } + si.values[string(k)] = &sv + return nil + }); err != nil { + return si, err + } + } + return si, nil +} + +func (s *StorageItem) ID() string { + return s.id +} + +func (s *StorageItem) View(fn func(b *bolt.Bucket) error) error { + return s.storage.View(s.id, fn) +} + +func (s *StorageItem) Update(fn func(b *bolt.Bucket) error) error { + return s.storage.Update(s.id, fn) +} + +func (s *StorageItem) Get(k string) *Value { + return s.values[k] +} + +func (s *StorageItem) Queue(fn func(b *bolt.Bucket) error) { + s.queue = append(s.queue, fn) +} + +func (s *StorageItem) Commit() error { + return s.Update(func(b *bolt.Bucket) error { + for _, fn := range s.queue { + if err := fn(b); err != nil { + return err + } + } + s.queue = s.queue[:0] + return nil + }) +} + +func (s *StorageItem) Indexes() (out []string) { + for _, v := range s.values { + if v.Index != "" { + out = append(out, v.Index) + } + } + return +} + +func (s *StorageItem) SetValue(b *bolt.Bucket, key string, v Value) error { + dt, err := json.Marshal(v) + if err != nil { + return err + } + if err := b.Put([]byte(key), dt); err != nil { + return err + } + if v.Index != "" { + b, err := b.Tx().CreateBucketIfNotExists([]byte(indexBucket)) + if err != nil { + return err + } + if err := b.Put([]byte(indexKey(v.Index, s.ID())), []byte{}); err != nil { + return err + } + } + s.values[key] = &v + return nil +} + +type Value struct { + Data []byte + Index string + // External bool +} + +func NewValue(v interface{}) (*Value, error) { + dt, err := json.Marshal(v) + if err != nil { + return nil, err + } + return &Value{Data: dt}, nil +} + +func (v *Value) Unmarshal(target interface{}) error { + err := json.Unmarshal(v.Data, target) + return err +} + +func indexKey(index, target string) string { + return index + "::" + target +} diff --git a/cache/metadata/metadata_test.go b/cache/metadata/metadata_test.go new file mode 100644 index 000000000000..0402422f82b1 --- /dev/null +++ b/cache/metadata/metadata_test.go @@ -0,0 +1,166 @@ +package metadata + +import ( + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/boltdb/bolt" + "github.com/stretchr/testify/require" +) + +func TestGetSetSearch(t *testing.T) { + tmpdir, err := ioutil.TempDir("", "buildkit-storage") + require.NoError(t, err) + defer os.RemoveAll(tmpdir) + + dbPath := filepath.Join(tmpdir, "storage.db") + + s, err := NewStore(dbPath) + require.NoError(t, err) + defer s.Close() + + si, ok := s.GetItem("foo") + require.False(t, ok) + + v := si.Get("bar") + require.Nil(t, v) + + v, err = NewValue("foobar") + require.NoError(t, err) + + si.Queue(func(b *bolt.Bucket) error { + return si.SetValue(b, "bar", *v) + }) + + err = si.Commit() + require.NoError(t, err) + + v = si.Get("bar") + require.NotNil(t, v) + + var str string + err = v.Unmarshal(&str) + require.NoError(t, err) + require.Equal(t, "foobar", str) + + err = s.Close() + require.NoError(t, err) + + s, err = NewStore(dbPath) + require.NoError(t, err) + defer s.Close() + + si, ok = s.GetItem("foo") + require.True(t, ok) + + v = si.Get("bar") + require.NotNil(t, v) + + str = "" + err = v.Unmarshal(&str) + require.NoError(t, err) + require.Equal(t, "foobar", str) + + // add second item to test Search + + si, ok = s.GetItem("foo2") + require.False(t, ok) + + v, err = NewValue("foobar2") + require.NoError(t, err) + + si.Queue(func(b *bolt.Bucket) error { + return si.SetValue(b, "bar2", *v) + }) + + err = si.Commit() + require.NoError(t, err) + + sis, err := s.All() + require.NoError(t, err) + require.Equal(t, 2, len(sis)) + + require.Equal(t, "foo", sis[0].ID()) + require.Equal(t, "foo2", sis[1].ID()) + + v = sis[0].Get("bar") + require.NotNil(t, v) + + str = "" + err = v.Unmarshal(&str) + require.NoError(t, err) + require.Equal(t, "foobar", str) + + // clear foo, check that only foo2 exists + err = s.Clear(sis[0].ID()) + require.NoError(t, err) + + sis, err = s.All() + require.NoError(t, err) + require.Equal(t, 1, len(sis)) + + require.Equal(t, "foo2", sis[0].ID()) + + _, ok = s.GetItem("foo") + require.False(t, ok) +} + +func TestIndexes(t *testing.T) { + tmpdir, err := ioutil.TempDir("", "buildkit-storage") + require.NoError(t, err) + defer os.RemoveAll(tmpdir) + + dbPath := filepath.Join(tmpdir, "storage.db") + + s, err := NewStore(dbPath) + require.NoError(t, err) + defer s.Close() + + var tcases = []struct { + key, valueKey, value, index string + }{ + {"foo1", "bar", "val1", "tag:baz"}, + {"foo2", "bar", "val2", "tag:bax"}, + {"foo3", "bar", "val3", "tag:baz"}, + } + + for _, tcase := range tcases { + si, ok := s.GetItem(tcase.key) + require.False(t, ok) + + v, err := NewValue(tcase.valueKey) + require.NoError(t, err) + v.Index = tcase.index + + si.Queue(func(b *bolt.Bucket) error { + return si.SetValue(b, tcase.value, *v) + }) + + err = si.Commit() + require.NoError(t, err) + } + + sis, err := s.Search("tag:baz") + require.NoError(t, err) + require.Equal(t, 2, len(sis)) + + require.Equal(t, sis[0].ID(), "foo1") + require.Equal(t, sis[1].ID(), "foo3") + + sis, err = s.Search("tag:bax") + require.NoError(t, err) + require.Equal(t, 1, len(sis)) + + require.Equal(t, sis[0].ID(), "foo2") + + err = s.Clear("foo1") + require.NoError(t, err) + + sis, err = s.Search("tag:baz") + require.NoError(t, err) + require.Equal(t, 1, len(sis)) + + require.Equal(t, sis[0].ID(), "foo3") +} diff --git a/cache/refs.go b/cache/refs.go index 41af54fc9aac..2e50ea78939b 100644 --- a/cache/refs.go +++ b/cache/refs.go @@ -4,14 +4,13 @@ import ( "sync" "github.com/containerd/containerd/mount" + "github.com/moby/buildkit/cache/metadata" "github.com/moby/buildkit/identity" "github.com/moby/buildkit/util/flightcontrol" "github.com/pkg/errors" "golang.org/x/net/context" ) -const sizeUnknown int64 = -1 - type ImmutableRef interface { Mountable ID() string @@ -38,14 +37,14 @@ type cacheRecord struct { frozen bool // meta SnapMeta refs map[Mountable]struct{} - id string cm *cacheManager parent ImmutableRef + md *metadata.StorageItem view string viewMount []mount.Mount sizeG flightcontrol.Group - size int64 + // size int64 } // hold manager lock before calling @@ -64,19 +63,22 @@ func (cr *cacheRecord) mref() *mutableRef { func (cr *cacheRecord) Size(ctx context.Context) (int64, error) { // this expects that usage() is implemented lazily - s, err := cr.sizeG.Do(ctx, cr.id, func(ctx context.Context) (interface{}, error) { + s, err := cr.sizeG.Do(ctx, cr.ID(), func(ctx context.Context) (interface{}, error) { cr.mu.Lock() - s := cr.size + s := getSize(cr.md) cr.mu.Unlock() if s != sizeUnknown { return s, nil } - usage, err := cr.cm.ManagerOpt.Snapshotter.Usage(ctx, cr.id) + usage, err := cr.cm.ManagerOpt.Snapshotter.Usage(ctx, cr.ID()) if err != nil { - return s, errors.Wrapf(err, "failed to get usage for %s", cr.id) + return s, errors.Wrapf(err, "failed to get usage for %s", cr.ID()) } cr.mu.Lock() - cr.size = usage.Size + setSize(cr.md, usage.Size) + if err := cr.md.Commit(); err != nil { + return s, err + } cr.mu.Unlock() return usage.Size, nil }) @@ -88,18 +90,18 @@ func (cr *cacheRecord) Mount(ctx context.Context) ([]mount.Mount, error) { defer cr.mu.Unlock() if cr.mutable { - m, err := cr.cm.Snapshotter.Mounts(ctx, cr.id) + m, err := cr.cm.Snapshotter.Mounts(ctx, cr.ID()) if err != nil { - return nil, errors.Wrapf(err, "failed to mount %s", cr.id) + return nil, errors.Wrapf(err, "failed to mount %s", cr.ID()) } return m, nil } if cr.viewMount == nil { // TODO: handle this better cr.view = identity.NewID() - m, err := cr.cm.Snapshotter.View(ctx, cr.view, cr.id) + m, err := cr.cm.Snapshotter.View(ctx, cr.view, cr.ID()) if err != nil { cr.view = "" - return nil, errors.Wrapf(err, "failed to mount %s", cr.id) + return nil, errors.Wrapf(err, "failed to mount %s", cr.ID()) } cr.viewMount = m } @@ -107,7 +109,7 @@ func (cr *cacheRecord) Mount(ctx context.Context) ([]mount.Mount, error) { } func (cr *cacheRecord) ID() string { - return cr.id + return cr.md.ID() } type immutableRef struct { @@ -167,7 +169,10 @@ func (sr *mutableRef) Freeze() (ImmutableRef, error) { sri := sr.ref() sri.frozen = true - sri.size = sizeUnknown + setSize(sr.md, sizeUnknown) + if err := sr.md.Commit(); err != nil { + return nil, err + } return sri, nil } @@ -191,19 +196,24 @@ func (sr *mutableRef) ReleaseAndCommit(ctx context.Context) (ImmutableRef, error id := identity.NewID() - err := sr.cm.Snapshotter.Commit(ctx, id, sr.id) + err := sr.cm.Snapshotter.Commit(ctx, id, sr.ID()) if err != nil { - return nil, errors.Wrapf(err, "failed to commit %s", sr.id) + return nil, errors.Wrapf(err, "failed to commit %s", sr.ID()) + } + + delete(sr.cm.records, sr.ID()) + + if err := sr.cm.md.Clear(sr.ID()); err != nil { + return nil, err } - delete(sr.cm.records, sr.id) + md, _ := sr.cm.md.Get(id) rec := &cacheRecord{ - id: id, cm: sr.cm, parent: sr.parent, refs: make(map[Mountable]struct{}), - size: sizeUnknown, + md: &md, } sr.cm.records[id] = rec // TODO: save to db From 44415841c984432eb1388d81da394f19d802a185 Mon Sep 17 00:00:00 2001 From: Tonis Tiigi <tonistiigi@gmail.com> Date: Tue, 4 Jul 2017 19:00:27 -0700 Subject: [PATCH 2/3] make blobmapping use metadata package Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com> --- cache/manager.go | 21 +---- cache/manager_test.go | 8 +- cache/metadata/metadata_test.go | 10 +- control/control_default.go | 16 +++- control/control_standalone_test.go | 14 ++- hack/test-unit | 2 +- snapshot/blobmapping/snapshotter.go | 136 ++++++++-------------------- 7 files changed, 72 insertions(+), 135 deletions(-) diff --git a/cache/manager.go b/cache/manager.go index 3a3f68679f8d..0ef2f69992a8 100644 --- a/cache/manager.go +++ b/cache/manager.go @@ -2,8 +2,6 @@ package cache import ( "context" - "os" - "path/filepath" "sync" "github.com/Sirupsen/logrus" @@ -16,8 +14,6 @@ import ( "golang.org/x/sync/errgroup" ) -const dbFile = "cache.db" - var ( errLocked = errors.New("locked") errNotFound = errors.New("not found") @@ -25,9 +21,9 @@ var ( ) type ManagerOpt struct { - Snapshotter snapshot.Snapshotter - Root string - GCPolicy GCPolicy + Snapshotter snapshot.Snapshotter + GCPolicy GCPolicy + MetadataStore *metadata.Store } type Accessor interface { @@ -56,18 +52,9 @@ type cacheManager struct { } func NewManager(opt ManagerOpt) (Manager, error) { - if err := os.MkdirAll(opt.Root, 0700); err != nil { - return nil, errors.Wrapf(err, "failed to create %s", opt.Root) - } - - md, err := metadata.NewStore(filepath.Join(opt.Root, dbFile)) - if err != nil { - return nil, err - } - cm := &cacheManager{ ManagerOpt: opt, - md: md, + md: opt.MetadataStore, records: make(map[string]*cacheRecord), } diff --git a/cache/manager_test.go b/cache/manager_test.go index 2ca1959ada94..80bb488d5aad 100644 --- a/cache/manager_test.go +++ b/cache/manager_test.go @@ -9,6 +9,7 @@ import ( "github.com/containerd/containerd/namespaces" "github.com/containerd/containerd/snapshot/naive" + "github.com/moby/buildkit/cache/metadata" "github.com/moby/buildkit/snapshot" "github.com/pkg/errors" "github.com/stretchr/testify/assert" @@ -24,9 +25,12 @@ func TestManager(t *testing.T) { snapshotter, err := naive.NewSnapshotter(filepath.Join(tmpdir, "snapshots")) assert.NoError(t, err) + md, err := metadata.NewStore(filepath.Join(tmpdir, "metadata.db")) + assert.NoError(t, err) + cm, err := NewManager(ManagerOpt{ - Root: tmpdir, - Snapshotter: snapshotter, + Snapshotter: snapshotter, + MetadataStore: md, }) assert.NoError(t, err) diff --git a/cache/metadata/metadata_test.go b/cache/metadata/metadata_test.go index 0402422f82b1..fec6781317f1 100644 --- a/cache/metadata/metadata_test.go +++ b/cache/metadata/metadata_test.go @@ -21,7 +21,7 @@ func TestGetSetSearch(t *testing.T) { require.NoError(t, err) defer s.Close() - si, ok := s.GetItem("foo") + si, ok := s.Get("foo") require.False(t, ok) v := si.Get("bar") @@ -52,7 +52,7 @@ func TestGetSetSearch(t *testing.T) { require.NoError(t, err) defer s.Close() - si, ok = s.GetItem("foo") + si, ok = s.Get("foo") require.True(t, ok) v = si.Get("bar") @@ -65,7 +65,7 @@ func TestGetSetSearch(t *testing.T) { // add second item to test Search - si, ok = s.GetItem("foo2") + si, ok = s.Get("foo2") require.False(t, ok) v, err = NewValue("foobar2") @@ -103,7 +103,7 @@ func TestGetSetSearch(t *testing.T) { require.Equal(t, "foo2", sis[0].ID()) - _, ok = s.GetItem("foo") + _, ok = s.Get("foo") require.False(t, ok) } @@ -127,7 +127,7 @@ func TestIndexes(t *testing.T) { } for _, tcase := range tcases { - si, ok := s.GetItem(tcase.key) + si, ok := s.Get(tcase.key) require.False(t, ok) v, err := NewValue(tcase.valueKey) diff --git a/control/control_default.go b/control/control_default.go index b0faa0b939bb..b9dd0ff84f03 100644 --- a/control/control_default.go +++ b/control/control_default.go @@ -9,6 +9,7 @@ import ( "github.com/containerd/containerd/rootfs" ctdsnapshot "github.com/containerd/containerd/snapshot" "github.com/moby/buildkit/cache" + "github.com/moby/buildkit/cache/metadata" "github.com/moby/buildkit/snapshot/blobmapping" "github.com/moby/buildkit/source" "github.com/moby/buildkit/source/containerimage" @@ -21,18 +22,23 @@ type pullDeps struct { } func defaultControllerOpts(root string, pd pullDeps) (*Opt, error) { + md, err := metadata.NewStore(filepath.Join(root, "metadata.db")) + if err != nil { + return nil, err + } + snapshotter, err := blobmapping.NewSnapshotter(blobmapping.Opt{ - Root: filepath.Join(root, "blobmap"), - Content: pd.ContentStore, - Snapshotter: pd.Snapshotter, + Content: pd.ContentStore, + Snapshotter: pd.Snapshotter, + MetadataStore: md, }) if err != nil { return nil, err } cm, err := cache.NewManager(cache.ManagerOpt{ - Snapshotter: snapshotter, - Root: filepath.Join(root, "cachemanager"), + Snapshotter: snapshotter, + MetadataStore: md, }) if err != nil { return nil, err diff --git a/control/control_standalone_test.go b/control/control_standalone_test.go index a5391f738322..2989200adbd9 100644 --- a/control/control_standalone_test.go +++ b/control/control_standalone_test.go @@ -11,6 +11,7 @@ import ( "github.com/containerd/containerd/namespaces" "github.com/moby/buildkit/cache" + "github.com/moby/buildkit/cache/metadata" "github.com/moby/buildkit/snapshot" "github.com/moby/buildkit/snapshot/blobmapping" "github.com/moby/buildkit/source" @@ -32,16 +33,19 @@ func TestControl(t *testing.T) { cd, err := newStandalonePullDeps(tmpdir) assert.NoError(t, err) + md, err := metadata.NewStore(filepath.Join(tmpdir, "metadata.db")) + assert.NoError(t, err) + snapshotter, err := blobmapping.NewSnapshotter(blobmapping.Opt{ - Root: filepath.Join(tmpdir, "blobmap"), - Content: cd.ContentStore, - Snapshotter: cd.Snapshotter, + Content: cd.ContentStore, + Snapshotter: cd.Snapshotter, + MetadataStore: md, }) assert.NoError(t, err) cm, err := cache.NewManager(cache.ManagerOpt{ - Snapshotter: snapshotter, - Root: filepath.Join(tmpdir, "cachemanager"), + Snapshotter: snapshotter, + MetadataStore: md, }) assert.NoError(t, err) diff --git a/hack/test-unit b/hack/test-unit index eff9819472f4..546966b13d6a 100755 --- a/hack/test-unit +++ b/hack/test-unit @@ -5,4 +5,4 @@ set -eu -o pipefail -x # update this to iidfile after 17.06 docker build -t buildkit:test --target unit-tests -f ./hack/dockerfiles/test.Dockerfile --force-rm . docker run --rm -v /tmp --privileged buildkit:test go test ${TESTFLAGS:--v} ${TESTPKGS:-./...} - +docker run --rm -v /tmp --privileged buildkit:test go test -tags standalone -v ./control diff --git a/snapshot/blobmapping/snapshotter.go b/snapshot/blobmapping/snapshotter.go index b6ab7499c9b2..79158a9a261f 100644 --- a/snapshot/blobmapping/snapshotter.go +++ b/snapshot/blobmapping/snapshotter.go @@ -1,29 +1,22 @@ package blobmapping import ( - "bytes" "context" - "os" - "path/filepath" + "github.com/Sirupsen/logrus" "github.com/boltdb/bolt" "github.com/containerd/containerd/content" "github.com/containerd/containerd/snapshot" + "github.com/moby/buildkit/cache/metadata" digest "github.com/opencontainers/go-digest" - "github.com/pkg/errors" ) -const dbFile = "blobmap.db" - -var ( - bucketBySnapshot = []byte("by_snapshot") - bucketByBlob = []byte("by_blob") -) +const blobKey = "blobmapping.blob" type Opt struct { - Content content.Store - Snapshotter snapshot.Snapshotter - Root string + Content content.Store + Snapshotter snapshot.Snapshotter + MetadataStore *metadata.Store } type Info struct { @@ -35,36 +28,18 @@ type Info struct { type Snapshotter struct { snapshot.Snapshotter - db *bolt.DB opt Opt } func NewSnapshotter(opt Opt) (*Snapshotter, error) { - if err := os.MkdirAll(opt.Root, 0700); err != nil { - return nil, errors.Wrapf(err, "failed to create %s", opt.Root) - } - - p := filepath.Join(opt.Root, dbFile) - db, err := bolt.Open(p, 0600, nil) - if err != nil { - return nil, errors.Wrapf(err, "failed to open database file %s", p) - } - s := &Snapshotter{ Snapshotter: opt.Snapshotter, - db: db, opt: opt, } return s, nil } -func (s *Snapshotter) init() error { - // this should do a walk from the DB and remove any records that are not - // in snapshotter any more - return nil -} - // Remove also removes a refrence to a blob. If it is a last reference then it deletes it the blob as well // Remove is not safe to be called concurrently func (s *Snapshotter) Remove(ctx context.Context, key string) error { @@ -73,26 +48,21 @@ func (s *Snapshotter) Remove(ctx context.Context, key string) error { return err } + blobs, err := s.opt.MetadataStore.Search(index(blob)) + if err != nil { + return err + } + if err := s.Snapshotter.Remove(ctx, key); err != nil { return err } - return s.db.Update(func(tx *bolt.Tx) error { - b := tx.Bucket(bucketBySnapshot) - if b == nil { - return nil - } - b.Delete([]byte(key)) - - if blob != "" { - b = tx.Bucket(bucketByBlob) - b.Delete(blobKey(blob, key)) - if len(keyRange(tx, blobKey(blob, ""))) == 0 { // last snapshot - s.opt.Content.Delete(ctx, blob) // log error - } + if len(blobs) == 1 && blobs[0].ID() == key { // last snapshot + if err := s.opt.Content.Delete(ctx, blob); err != nil { + logrus.Errorf("failed to delete blob %v", blob) } - return nil - }) + } + return nil } func (s *Snapshotter) Usage(ctx context.Context, key string) (snapshot.Usage, error) { @@ -114,23 +84,17 @@ func (s *Snapshotter) Usage(ctx context.Context, key string) (snapshot.Usage, er return u, nil } -// TODO: make Blob/SetBlob part of generic metadata wrapper that can detect -// blob key for deletion logic - func (s *Snapshotter) GetBlob(ctx context.Context, key string) (digest.Digest, error) { + md, _ := s.opt.MetadataStore.Get(key) + v := md.Get(blobKey) + if v == nil { + return "", nil + } var blob digest.Digest - err := s.db.View(func(tx *bolt.Tx) error { - b := tx.Bucket(bucketBySnapshot) - if b == nil { - return nil - } - v := b.Get([]byte(key)) - if v != nil { - blob = digest.Digest(v) - } - return nil - }) - return blob, err + if err := v.Unmarshal(&blob); err != nil { + return "", err + } + return blob, nil } // Validates that there is no blob associated with the snapshot. @@ -141,47 +105,19 @@ func (s *Snapshotter) SetBlob(ctx context.Context, key string, blob digest.Diges if err != nil { return err } - return s.db.Update(func(tx *bolt.Tx) error { - b, err := tx.CreateBucketIfNotExists(bucketBySnapshot) - if err != nil { - return err - } - v := b.Get([]byte(key)) - if v != nil { - if string(v) != string(blob) { - return errors.Errorf("different blob already set for %s", key) - } else { - return nil - } - } + md, _ := s.opt.MetadataStore.Get(key) - if err := b.Put([]byte(key), []byte(blob)); err != nil { - return err - } - b, err = tx.CreateBucketIfNotExists(bucketByBlob) - if err != nil { - return err - } - return b.Put(blobKey(blob, key), []byte{}) - }) -} + v, err := metadata.NewValue(blob) + if err != nil { + return err + } + v.Index = index(blob) -func blobKey(blob digest.Digest, snapshot string) []byte { - return []byte(string(blob) + "-" + snapshot) + return md.Update(func(b *bolt.Bucket) error { + return md.SetValue(b, blobKey, *v) + }) } -// results are only valid for the lifetime of the transaction -func keyRange(tx *bolt.Tx, key []byte) (out [][]byte) { - c := tx.Cursor() - lastKey := append([]byte{}, key...) - lastKey = append(lastKey, ^byte(0)) - k, _ := c.Seek([]byte(key)) - for { - if k != nil && bytes.Compare(k, lastKey) <= 0 { - out = append(out, k) - continue - } - break - } - return +func index(blob digest.Digest) string { + return "blobmap::" + blob.String() } From c28c8c0789062b7ece00786b5fdfe2665e7c48be Mon Sep 17 00:00:00 2001 From: Tonis Tiigi <tonistiigi@gmail.com> Date: Tue, 4 Jul 2017 19:00:37 -0700 Subject: [PATCH 3/3] vendor: add require Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com> --- .../stretchr/testify/require/doc.go | 28 ++ .../testify/require/forward_requirements.go | 16 + .../stretchr/testify/require/require.go | 464 ++++++++++++++++++ .../testify/require/require_forward.go | 388 +++++++++++++++ .../stretchr/testify/require/requirements.go | 9 + 5 files changed, 905 insertions(+) create mode 100644 vendor/github.com/stretchr/testify/require/doc.go create mode 100644 vendor/github.com/stretchr/testify/require/forward_requirements.go create mode 100644 vendor/github.com/stretchr/testify/require/require.go create mode 100644 vendor/github.com/stretchr/testify/require/require_forward.go create mode 100644 vendor/github.com/stretchr/testify/require/requirements.go diff --git a/vendor/github.com/stretchr/testify/require/doc.go b/vendor/github.com/stretchr/testify/require/doc.go new file mode 100644 index 000000000000..169de39221c7 --- /dev/null +++ b/vendor/github.com/stretchr/testify/require/doc.go @@ -0,0 +1,28 @@ +// Package require implements the same assertions as the `assert` package but +// stops test execution when a test fails. +// +// Example Usage +// +// The following is a complete example using require in a standard test function: +// import ( +// "testing" +// "github.com/stretchr/testify/require" +// ) +// +// func TestSomething(t *testing.T) { +// +// var a string = "Hello" +// var b string = "Hello" +// +// require.Equal(t, a, b, "The two words should be the same.") +// +// } +// +// Assertions +// +// The `require` package have same global functions as in the `assert` package, +// but instead of returning a boolean result they call `t.FailNow()`. +// +// Every assertion function also takes an optional string message as the final argument, +// allowing custom error messages to be appended to the message the assertion method outputs. +package require diff --git a/vendor/github.com/stretchr/testify/require/forward_requirements.go b/vendor/github.com/stretchr/testify/require/forward_requirements.go new file mode 100644 index 000000000000..d3c2ab9bc7eb --- /dev/null +++ b/vendor/github.com/stretchr/testify/require/forward_requirements.go @@ -0,0 +1,16 @@ +package require + +// Assertions provides assertion methods around the +// TestingT interface. +type Assertions struct { + t TestingT +} + +// New makes a new Assertions object for the specified TestingT. +func New(t TestingT) *Assertions { + return &Assertions{ + t: t, + } +} + +//go:generate go run ../_codegen/main.go -output-package=require -template=require_forward.go.tmpl diff --git a/vendor/github.com/stretchr/testify/require/require.go b/vendor/github.com/stretchr/testify/require/require.go new file mode 100644 index 000000000000..1bcfcb0d949d --- /dev/null +++ b/vendor/github.com/stretchr/testify/require/require.go @@ -0,0 +1,464 @@ +/* +* CODE GENERATED AUTOMATICALLY WITH github.com/stretchr/testify/_codegen +* THIS FILE MUST NOT BE EDITED BY HAND +*/ + +package require + +import ( + + assert "github.com/stretchr/testify/assert" + http "net/http" + url "net/url" + time "time" +) + + +// Condition uses a Comparison to assert a complex condition. +func Condition(t TestingT, comp assert.Comparison, msgAndArgs ...interface{}) { + if !assert.Condition(t, comp, msgAndArgs...) { + t.FailNow() + } +} + + +// Contains asserts that the specified string, list(array, slice...) or map contains the +// specified substring or element. +// +// assert.Contains(t, "Hello World", "World", "But 'Hello World' does contain 'World'") +// assert.Contains(t, ["Hello", "World"], "World", "But ["Hello", "World"] does contain 'World'") +// assert.Contains(t, {"Hello": "World"}, "Hello", "But {'Hello': 'World'} does contain 'Hello'") +// +// Returns whether the assertion was successful (true) or not (false). +func Contains(t TestingT, s interface{}, contains interface{}, msgAndArgs ...interface{}) { + if !assert.Contains(t, s, contains, msgAndArgs...) { + t.FailNow() + } +} + + +// Empty asserts that the specified object is empty. I.e. nil, "", false, 0 or either +// a slice or a channel with len == 0. +// +// assert.Empty(t, obj) +// +// Returns whether the assertion was successful (true) or not (false). +func Empty(t TestingT, object interface{}, msgAndArgs ...interface{}) { + if !assert.Empty(t, object, msgAndArgs...) { + t.FailNow() + } +} + + +// Equal asserts that two objects are equal. +// +// assert.Equal(t, 123, 123, "123 and 123 should be equal") +// +// Returns whether the assertion was successful (true) or not (false). +func Equal(t TestingT, expected interface{}, actual interface{}, msgAndArgs ...interface{}) { + if !assert.Equal(t, expected, actual, msgAndArgs...) { + t.FailNow() + } +} + + +// EqualError asserts that a function returned an error (i.e. not `nil`) +// and that it is equal to the provided error. +// +// actualObj, err := SomeFunction() +// if assert.Error(t, err, "An error was expected") { +// assert.Equal(t, err, expectedError) +// } +// +// Returns whether the assertion was successful (true) or not (false). +func EqualError(t TestingT, theError error, errString string, msgAndArgs ...interface{}) { + if !assert.EqualError(t, theError, errString, msgAndArgs...) { + t.FailNow() + } +} + + +// EqualValues asserts that two objects are equal or convertable to the same types +// and equal. +// +// assert.EqualValues(t, uint32(123), int32(123), "123 and 123 should be equal") +// +// Returns whether the assertion was successful (true) or not (false). +func EqualValues(t TestingT, expected interface{}, actual interface{}, msgAndArgs ...interface{}) { + if !assert.EqualValues(t, expected, actual, msgAndArgs...) { + t.FailNow() + } +} + + +// Error asserts that a function returned an error (i.e. not `nil`). +// +// actualObj, err := SomeFunction() +// if assert.Error(t, err, "An error was expected") { +// assert.Equal(t, err, expectedError) +// } +// +// Returns whether the assertion was successful (true) or not (false). +func Error(t TestingT, err error, msgAndArgs ...interface{}) { + if !assert.Error(t, err, msgAndArgs...) { + t.FailNow() + } +} + + +// Exactly asserts that two objects are equal is value and type. +// +// assert.Exactly(t, int32(123), int64(123), "123 and 123 should NOT be equal") +// +// Returns whether the assertion was successful (true) or not (false). +func Exactly(t TestingT, expected interface{}, actual interface{}, msgAndArgs ...interface{}) { + if !assert.Exactly(t, expected, actual, msgAndArgs...) { + t.FailNow() + } +} + + +// Fail reports a failure through +func Fail(t TestingT, failureMessage string, msgAndArgs ...interface{}) { + if !assert.Fail(t, failureMessage, msgAndArgs...) { + t.FailNow() + } +} + + +// FailNow fails test +func FailNow(t TestingT, failureMessage string, msgAndArgs ...interface{}) { + if !assert.FailNow(t, failureMessage, msgAndArgs...) { + t.FailNow() + } +} + + +// False asserts that the specified value is false. +// +// assert.False(t, myBool, "myBool should be false") +// +// Returns whether the assertion was successful (true) or not (false). +func False(t TestingT, value bool, msgAndArgs ...interface{}) { + if !assert.False(t, value, msgAndArgs...) { + t.FailNow() + } +} + + +// HTTPBodyContains asserts that a specified handler returns a +// body that contains a string. +// +// assert.HTTPBodyContains(t, myHandler, "www.google.com", nil, "I'm Feeling Lucky") +// +// Returns whether the assertion was successful (true) or not (false). +func HTTPBodyContains(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values, str interface{}) { + if !assert.HTTPBodyContains(t, handler, method, url, values, str) { + t.FailNow() + } +} + + +// HTTPBodyNotContains asserts that a specified handler returns a +// body that does not contain a string. +// +// assert.HTTPBodyNotContains(t, myHandler, "www.google.com", nil, "I'm Feeling Lucky") +// +// Returns whether the assertion was successful (true) or not (false). +func HTTPBodyNotContains(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values, str interface{}) { + if !assert.HTTPBodyNotContains(t, handler, method, url, values, str) { + t.FailNow() + } +} + + +// HTTPError asserts that a specified handler returns an error status code. +// +// assert.HTTPError(t, myHandler, "POST", "/a/b/c", url.Values{"a": []string{"b", "c"}} +// +// Returns whether the assertion was successful (true) or not (false). +func HTTPError(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values) { + if !assert.HTTPError(t, handler, method, url, values) { + t.FailNow() + } +} + + +// HTTPRedirect asserts that a specified handler returns a redirect status code. +// +// assert.HTTPRedirect(t, myHandler, "GET", "/a/b/c", url.Values{"a": []string{"b", "c"}} +// +// Returns whether the assertion was successful (true) or not (false). +func HTTPRedirect(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values) { + if !assert.HTTPRedirect(t, handler, method, url, values) { + t.FailNow() + } +} + + +// HTTPSuccess asserts that a specified handler returns a success status code. +// +// assert.HTTPSuccess(t, myHandler, "POST", "http://www.google.com", nil) +// +// Returns whether the assertion was successful (true) or not (false). +func HTTPSuccess(t TestingT, handler http.HandlerFunc, method string, url string, values url.Values) { + if !assert.HTTPSuccess(t, handler, method, url, values) { + t.FailNow() + } +} + + +// Implements asserts that an object is implemented by the specified interface. +// +// assert.Implements(t, (*MyInterface)(nil), new(MyObject), "MyObject") +func Implements(t TestingT, interfaceObject interface{}, object interface{}, msgAndArgs ...interface{}) { + if !assert.Implements(t, interfaceObject, object, msgAndArgs...) { + t.FailNow() + } +} + + +// InDelta asserts that the two numerals are within delta of each other. +// +// assert.InDelta(t, math.Pi, (22 / 7.0), 0.01) +// +// Returns whether the assertion was successful (true) or not (false). +func InDelta(t TestingT, expected interface{}, actual interface{}, delta float64, msgAndArgs ...interface{}) { + if !assert.InDelta(t, expected, actual, delta, msgAndArgs...) { + t.FailNow() + } +} + + +// InDeltaSlice is the same as InDelta, except it compares two slices. +func InDeltaSlice(t TestingT, expected interface{}, actual interface{}, delta float64, msgAndArgs ...interface{}) { + if !assert.InDeltaSlice(t, expected, actual, delta, msgAndArgs...) { + t.FailNow() + } +} + + +// InEpsilon asserts that expected and actual have a relative error less than epsilon +// +// Returns whether the assertion was successful (true) or not (false). +func InEpsilon(t TestingT, expected interface{}, actual interface{}, epsilon float64, msgAndArgs ...interface{}) { + if !assert.InEpsilon(t, expected, actual, epsilon, msgAndArgs...) { + t.FailNow() + } +} + + +// InEpsilonSlice is the same as InEpsilon, except it compares two slices. +func InEpsilonSlice(t TestingT, expected interface{}, actual interface{}, delta float64, msgAndArgs ...interface{}) { + if !assert.InEpsilonSlice(t, expected, actual, delta, msgAndArgs...) { + t.FailNow() + } +} + + +// IsType asserts that the specified objects are of the same type. +func IsType(t TestingT, expectedType interface{}, object interface{}, msgAndArgs ...interface{}) { + if !assert.IsType(t, expectedType, object, msgAndArgs...) { + t.FailNow() + } +} + + +// JSONEq asserts that two JSON strings are equivalent. +// +// assert.JSONEq(t, `{"hello": "world", "foo": "bar"}`, `{"foo": "bar", "hello": "world"}`) +// +// Returns whether the assertion was successful (true) or not (false). +func JSONEq(t TestingT, expected string, actual string, msgAndArgs ...interface{}) { + if !assert.JSONEq(t, expected, actual, msgAndArgs...) { + t.FailNow() + } +} + + +// Len asserts that the specified object has specific length. +// Len also fails if the object has a type that len() not accept. +// +// assert.Len(t, mySlice, 3, "The size of slice is not 3") +// +// Returns whether the assertion was successful (true) or not (false). +func Len(t TestingT, object interface{}, length int, msgAndArgs ...interface{}) { + if !assert.Len(t, object, length, msgAndArgs...) { + t.FailNow() + } +} + + +// Nil asserts that the specified object is nil. +// +// assert.Nil(t, err, "err should be nothing") +// +// Returns whether the assertion was successful (true) or not (false). +func Nil(t TestingT, object interface{}, msgAndArgs ...interface{}) { + if !assert.Nil(t, object, msgAndArgs...) { + t.FailNow() + } +} + + +// NoError asserts that a function returned no error (i.e. `nil`). +// +// actualObj, err := SomeFunction() +// if assert.NoError(t, err) { +// assert.Equal(t, actualObj, expectedObj) +// } +// +// Returns whether the assertion was successful (true) or not (false). +func NoError(t TestingT, err error, msgAndArgs ...interface{}) { + if !assert.NoError(t, err, msgAndArgs...) { + t.FailNow() + } +} + + +// NotContains asserts that the specified string, list(array, slice...) or map does NOT contain the +// specified substring or element. +// +// assert.NotContains(t, "Hello World", "Earth", "But 'Hello World' does NOT contain 'Earth'") +// assert.NotContains(t, ["Hello", "World"], "Earth", "But ['Hello', 'World'] does NOT contain 'Earth'") +// assert.NotContains(t, {"Hello": "World"}, "Earth", "But {'Hello': 'World'} does NOT contain 'Earth'") +// +// Returns whether the assertion was successful (true) or not (false). +func NotContains(t TestingT, s interface{}, contains interface{}, msgAndArgs ...interface{}) { + if !assert.NotContains(t, s, contains, msgAndArgs...) { + t.FailNow() + } +} + + +// NotEmpty asserts that the specified object is NOT empty. I.e. not nil, "", false, 0 or either +// a slice or a channel with len == 0. +// +// if assert.NotEmpty(t, obj) { +// assert.Equal(t, "two", obj[1]) +// } +// +// Returns whether the assertion was successful (true) or not (false). +func NotEmpty(t TestingT, object interface{}, msgAndArgs ...interface{}) { + if !assert.NotEmpty(t, object, msgAndArgs...) { + t.FailNow() + } +} + + +// NotEqual asserts that the specified values are NOT equal. +// +// assert.NotEqual(t, obj1, obj2, "two objects shouldn't be equal") +// +// Returns whether the assertion was successful (true) or not (false). +func NotEqual(t TestingT, expected interface{}, actual interface{}, msgAndArgs ...interface{}) { + if !assert.NotEqual(t, expected, actual, msgAndArgs...) { + t.FailNow() + } +} + + +// NotNil asserts that the specified object is not nil. +// +// assert.NotNil(t, err, "err should be something") +// +// Returns whether the assertion was successful (true) or not (false). +func NotNil(t TestingT, object interface{}, msgAndArgs ...interface{}) { + if !assert.NotNil(t, object, msgAndArgs...) { + t.FailNow() + } +} + + +// NotPanics asserts that the code inside the specified PanicTestFunc does NOT panic. +// +// assert.NotPanics(t, func(){ +// RemainCalm() +// }, "Calling RemainCalm() should NOT panic") +// +// Returns whether the assertion was successful (true) or not (false). +func NotPanics(t TestingT, f assert.PanicTestFunc, msgAndArgs ...interface{}) { + if !assert.NotPanics(t, f, msgAndArgs...) { + t.FailNow() + } +} + + +// NotRegexp asserts that a specified regexp does not match a string. +// +// assert.NotRegexp(t, regexp.MustCompile("starts"), "it's starting") +// assert.NotRegexp(t, "^start", "it's not starting") +// +// Returns whether the assertion was successful (true) or not (false). +func NotRegexp(t TestingT, rx interface{}, str interface{}, msgAndArgs ...interface{}) { + if !assert.NotRegexp(t, rx, str, msgAndArgs...) { + t.FailNow() + } +} + + +// NotZero asserts that i is not the zero value for its type and returns the truth. +func NotZero(t TestingT, i interface{}, msgAndArgs ...interface{}) { + if !assert.NotZero(t, i, msgAndArgs...) { + t.FailNow() + } +} + + +// Panics asserts that the code inside the specified PanicTestFunc panics. +// +// assert.Panics(t, func(){ +// GoCrazy() +// }, "Calling GoCrazy() should panic") +// +// Returns whether the assertion was successful (true) or not (false). +func Panics(t TestingT, f assert.PanicTestFunc, msgAndArgs ...interface{}) { + if !assert.Panics(t, f, msgAndArgs...) { + t.FailNow() + } +} + + +// Regexp asserts that a specified regexp matches a string. +// +// assert.Regexp(t, regexp.MustCompile("start"), "it's starting") +// assert.Regexp(t, "start...$", "it's not starting") +// +// Returns whether the assertion was successful (true) or not (false). +func Regexp(t TestingT, rx interface{}, str interface{}, msgAndArgs ...interface{}) { + if !assert.Regexp(t, rx, str, msgAndArgs...) { + t.FailNow() + } +} + + +// True asserts that the specified value is true. +// +// assert.True(t, myBool, "myBool should be true") +// +// Returns whether the assertion was successful (true) or not (false). +func True(t TestingT, value bool, msgAndArgs ...interface{}) { + if !assert.True(t, value, msgAndArgs...) { + t.FailNow() + } +} + + +// WithinDuration asserts that the two times are within duration delta of each other. +// +// assert.WithinDuration(t, time.Now(), time.Now(), 10*time.Second, "The difference should not be more than 10s") +// +// Returns whether the assertion was successful (true) or not (false). +func WithinDuration(t TestingT, expected time.Time, actual time.Time, delta time.Duration, msgAndArgs ...interface{}) { + if !assert.WithinDuration(t, expected, actual, delta, msgAndArgs...) { + t.FailNow() + } +} + + +// Zero asserts that i is the zero value for its type and returns the truth. +func Zero(t TestingT, i interface{}, msgAndArgs ...interface{}) { + if !assert.Zero(t, i, msgAndArgs...) { + t.FailNow() + } +} diff --git a/vendor/github.com/stretchr/testify/require/require_forward.go b/vendor/github.com/stretchr/testify/require/require_forward.go new file mode 100644 index 000000000000..58324f10551c --- /dev/null +++ b/vendor/github.com/stretchr/testify/require/require_forward.go @@ -0,0 +1,388 @@ +/* +* CODE GENERATED AUTOMATICALLY WITH github.com/stretchr/testify/_codegen +* THIS FILE MUST NOT BE EDITED BY HAND +*/ + +package require + +import ( + + assert "github.com/stretchr/testify/assert" + http "net/http" + url "net/url" + time "time" +) + + +// Condition uses a Comparison to assert a complex condition. +func (a *Assertions) Condition(comp assert.Comparison, msgAndArgs ...interface{}) { + Condition(a.t, comp, msgAndArgs...) +} + + +// Contains asserts that the specified string, list(array, slice...) or map contains the +// specified substring or element. +// +// a.Contains("Hello World", "World", "But 'Hello World' does contain 'World'") +// a.Contains(["Hello", "World"], "World", "But ["Hello", "World"] does contain 'World'") +// a.Contains({"Hello": "World"}, "Hello", "But {'Hello': 'World'} does contain 'Hello'") +// +// Returns whether the assertion was successful (true) or not (false). +func (a *Assertions) Contains(s interface{}, contains interface{}, msgAndArgs ...interface{}) { + Contains(a.t, s, contains, msgAndArgs...) +} + + +// Empty asserts that the specified object is empty. I.e. nil, "", false, 0 or either +// a slice or a channel with len == 0. +// +// a.Empty(obj) +// +// Returns whether the assertion was successful (true) or not (false). +func (a *Assertions) Empty(object interface{}, msgAndArgs ...interface{}) { + Empty(a.t, object, msgAndArgs...) +} + + +// Equal asserts that two objects are equal. +// +// a.Equal(123, 123, "123 and 123 should be equal") +// +// Returns whether the assertion was successful (true) or not (false). +func (a *Assertions) Equal(expected interface{}, actual interface{}, msgAndArgs ...interface{}) { + Equal(a.t, expected, actual, msgAndArgs...) +} + + +// EqualError asserts that a function returned an error (i.e. not `nil`) +// and that it is equal to the provided error. +// +// actualObj, err := SomeFunction() +// if assert.Error(t, err, "An error was expected") { +// assert.Equal(t, err, expectedError) +// } +// +// Returns whether the assertion was successful (true) or not (false). +func (a *Assertions) EqualError(theError error, errString string, msgAndArgs ...interface{}) { + EqualError(a.t, theError, errString, msgAndArgs...) +} + + +// EqualValues asserts that two objects are equal or convertable to the same types +// and equal. +// +// a.EqualValues(uint32(123), int32(123), "123 and 123 should be equal") +// +// Returns whether the assertion was successful (true) or not (false). +func (a *Assertions) EqualValues(expected interface{}, actual interface{}, msgAndArgs ...interface{}) { + EqualValues(a.t, expected, actual, msgAndArgs...) +} + + +// Error asserts that a function returned an error (i.e. not `nil`). +// +// actualObj, err := SomeFunction() +// if a.Error(err, "An error was expected") { +// assert.Equal(t, err, expectedError) +// } +// +// Returns whether the assertion was successful (true) or not (false). +func (a *Assertions) Error(err error, msgAndArgs ...interface{}) { + Error(a.t, err, msgAndArgs...) +} + + +// Exactly asserts that two objects are equal is value and type. +// +// a.Exactly(int32(123), int64(123), "123 and 123 should NOT be equal") +// +// Returns whether the assertion was successful (true) or not (false). +func (a *Assertions) Exactly(expected interface{}, actual interface{}, msgAndArgs ...interface{}) { + Exactly(a.t, expected, actual, msgAndArgs...) +} + + +// Fail reports a failure through +func (a *Assertions) Fail(failureMessage string, msgAndArgs ...interface{}) { + Fail(a.t, failureMessage, msgAndArgs...) +} + + +// FailNow fails test +func (a *Assertions) FailNow(failureMessage string, msgAndArgs ...interface{}) { + FailNow(a.t, failureMessage, msgAndArgs...) +} + + +// False asserts that the specified value is false. +// +// a.False(myBool, "myBool should be false") +// +// Returns whether the assertion was successful (true) or not (false). +func (a *Assertions) False(value bool, msgAndArgs ...interface{}) { + False(a.t, value, msgAndArgs...) +} + + +// HTTPBodyContains asserts that a specified handler returns a +// body that contains a string. +// +// a.HTTPBodyContains(myHandler, "www.google.com", nil, "I'm Feeling Lucky") +// +// Returns whether the assertion was successful (true) or not (false). +func (a *Assertions) HTTPBodyContains(handler http.HandlerFunc, method string, url string, values url.Values, str interface{}) { + HTTPBodyContains(a.t, handler, method, url, values, str) +} + + +// HTTPBodyNotContains asserts that a specified handler returns a +// body that does not contain a string. +// +// a.HTTPBodyNotContains(myHandler, "www.google.com", nil, "I'm Feeling Lucky") +// +// Returns whether the assertion was successful (true) or not (false). +func (a *Assertions) HTTPBodyNotContains(handler http.HandlerFunc, method string, url string, values url.Values, str interface{}) { + HTTPBodyNotContains(a.t, handler, method, url, values, str) +} + + +// HTTPError asserts that a specified handler returns an error status code. +// +// a.HTTPError(myHandler, "POST", "/a/b/c", url.Values{"a": []string{"b", "c"}} +// +// Returns whether the assertion was successful (true) or not (false). +func (a *Assertions) HTTPError(handler http.HandlerFunc, method string, url string, values url.Values) { + HTTPError(a.t, handler, method, url, values) +} + + +// HTTPRedirect asserts that a specified handler returns a redirect status code. +// +// a.HTTPRedirect(myHandler, "GET", "/a/b/c", url.Values{"a": []string{"b", "c"}} +// +// Returns whether the assertion was successful (true) or not (false). +func (a *Assertions) HTTPRedirect(handler http.HandlerFunc, method string, url string, values url.Values) { + HTTPRedirect(a.t, handler, method, url, values) +} + + +// HTTPSuccess asserts that a specified handler returns a success status code. +// +// a.HTTPSuccess(myHandler, "POST", "http://www.google.com", nil) +// +// Returns whether the assertion was successful (true) or not (false). +func (a *Assertions) HTTPSuccess(handler http.HandlerFunc, method string, url string, values url.Values) { + HTTPSuccess(a.t, handler, method, url, values) +} + + +// Implements asserts that an object is implemented by the specified interface. +// +// a.Implements((*MyInterface)(nil), new(MyObject), "MyObject") +func (a *Assertions) Implements(interfaceObject interface{}, object interface{}, msgAndArgs ...interface{}) { + Implements(a.t, interfaceObject, object, msgAndArgs...) +} + + +// InDelta asserts that the two numerals are within delta of each other. +// +// a.InDelta(math.Pi, (22 / 7.0), 0.01) +// +// Returns whether the assertion was successful (true) or not (false). +func (a *Assertions) InDelta(expected interface{}, actual interface{}, delta float64, msgAndArgs ...interface{}) { + InDelta(a.t, expected, actual, delta, msgAndArgs...) +} + + +// InDeltaSlice is the same as InDelta, except it compares two slices. +func (a *Assertions) InDeltaSlice(expected interface{}, actual interface{}, delta float64, msgAndArgs ...interface{}) { + InDeltaSlice(a.t, expected, actual, delta, msgAndArgs...) +} + + +// InEpsilon asserts that expected and actual have a relative error less than epsilon +// +// Returns whether the assertion was successful (true) or not (false). +func (a *Assertions) InEpsilon(expected interface{}, actual interface{}, epsilon float64, msgAndArgs ...interface{}) { + InEpsilon(a.t, expected, actual, epsilon, msgAndArgs...) +} + + +// InEpsilonSlice is the same as InEpsilon, except it compares two slices. +func (a *Assertions) InEpsilonSlice(expected interface{}, actual interface{}, delta float64, msgAndArgs ...interface{}) { + InEpsilonSlice(a.t, expected, actual, delta, msgAndArgs...) +} + + +// IsType asserts that the specified objects are of the same type. +func (a *Assertions) IsType(expectedType interface{}, object interface{}, msgAndArgs ...interface{}) { + IsType(a.t, expectedType, object, msgAndArgs...) +} + + +// JSONEq asserts that two JSON strings are equivalent. +// +// a.JSONEq(`{"hello": "world", "foo": "bar"}`, `{"foo": "bar", "hello": "world"}`) +// +// Returns whether the assertion was successful (true) or not (false). +func (a *Assertions) JSONEq(expected string, actual string, msgAndArgs ...interface{}) { + JSONEq(a.t, expected, actual, msgAndArgs...) +} + + +// Len asserts that the specified object has specific length. +// Len also fails if the object has a type that len() not accept. +// +// a.Len(mySlice, 3, "The size of slice is not 3") +// +// Returns whether the assertion was successful (true) or not (false). +func (a *Assertions) Len(object interface{}, length int, msgAndArgs ...interface{}) { + Len(a.t, object, length, msgAndArgs...) +} + + +// Nil asserts that the specified object is nil. +// +// a.Nil(err, "err should be nothing") +// +// Returns whether the assertion was successful (true) or not (false). +func (a *Assertions) Nil(object interface{}, msgAndArgs ...interface{}) { + Nil(a.t, object, msgAndArgs...) +} + + +// NoError asserts that a function returned no error (i.e. `nil`). +// +// actualObj, err := SomeFunction() +// if a.NoError(err) { +// assert.Equal(t, actualObj, expectedObj) +// } +// +// Returns whether the assertion was successful (true) or not (false). +func (a *Assertions) NoError(err error, msgAndArgs ...interface{}) { + NoError(a.t, err, msgAndArgs...) +} + + +// NotContains asserts that the specified string, list(array, slice...) or map does NOT contain the +// specified substring or element. +// +// a.NotContains("Hello World", "Earth", "But 'Hello World' does NOT contain 'Earth'") +// a.NotContains(["Hello", "World"], "Earth", "But ['Hello', 'World'] does NOT contain 'Earth'") +// a.NotContains({"Hello": "World"}, "Earth", "But {'Hello': 'World'} does NOT contain 'Earth'") +// +// Returns whether the assertion was successful (true) or not (false). +func (a *Assertions) NotContains(s interface{}, contains interface{}, msgAndArgs ...interface{}) { + NotContains(a.t, s, contains, msgAndArgs...) +} + + +// NotEmpty asserts that the specified object is NOT empty. I.e. not nil, "", false, 0 or either +// a slice or a channel with len == 0. +// +// if a.NotEmpty(obj) { +// assert.Equal(t, "two", obj[1]) +// } +// +// Returns whether the assertion was successful (true) or not (false). +func (a *Assertions) NotEmpty(object interface{}, msgAndArgs ...interface{}) { + NotEmpty(a.t, object, msgAndArgs...) +} + + +// NotEqual asserts that the specified values are NOT equal. +// +// a.NotEqual(obj1, obj2, "two objects shouldn't be equal") +// +// Returns whether the assertion was successful (true) or not (false). +func (a *Assertions) NotEqual(expected interface{}, actual interface{}, msgAndArgs ...interface{}) { + NotEqual(a.t, expected, actual, msgAndArgs...) +} + + +// NotNil asserts that the specified object is not nil. +// +// a.NotNil(err, "err should be something") +// +// Returns whether the assertion was successful (true) or not (false). +func (a *Assertions) NotNil(object interface{}, msgAndArgs ...interface{}) { + NotNil(a.t, object, msgAndArgs...) +} + + +// NotPanics asserts that the code inside the specified PanicTestFunc does NOT panic. +// +// a.NotPanics(func(){ +// RemainCalm() +// }, "Calling RemainCalm() should NOT panic") +// +// Returns whether the assertion was successful (true) or not (false). +func (a *Assertions) NotPanics(f assert.PanicTestFunc, msgAndArgs ...interface{}) { + NotPanics(a.t, f, msgAndArgs...) +} + + +// NotRegexp asserts that a specified regexp does not match a string. +// +// a.NotRegexp(regexp.MustCompile("starts"), "it's starting") +// a.NotRegexp("^start", "it's not starting") +// +// Returns whether the assertion was successful (true) or not (false). +func (a *Assertions) NotRegexp(rx interface{}, str interface{}, msgAndArgs ...interface{}) { + NotRegexp(a.t, rx, str, msgAndArgs...) +} + + +// NotZero asserts that i is not the zero value for its type and returns the truth. +func (a *Assertions) NotZero(i interface{}, msgAndArgs ...interface{}) { + NotZero(a.t, i, msgAndArgs...) +} + + +// Panics asserts that the code inside the specified PanicTestFunc panics. +// +// a.Panics(func(){ +// GoCrazy() +// }, "Calling GoCrazy() should panic") +// +// Returns whether the assertion was successful (true) or not (false). +func (a *Assertions) Panics(f assert.PanicTestFunc, msgAndArgs ...interface{}) { + Panics(a.t, f, msgAndArgs...) +} + + +// Regexp asserts that a specified regexp matches a string. +// +// a.Regexp(regexp.MustCompile("start"), "it's starting") +// a.Regexp("start...$", "it's not starting") +// +// Returns whether the assertion was successful (true) or not (false). +func (a *Assertions) Regexp(rx interface{}, str interface{}, msgAndArgs ...interface{}) { + Regexp(a.t, rx, str, msgAndArgs...) +} + + +// True asserts that the specified value is true. +// +// a.True(myBool, "myBool should be true") +// +// Returns whether the assertion was successful (true) or not (false). +func (a *Assertions) True(value bool, msgAndArgs ...interface{}) { + True(a.t, value, msgAndArgs...) +} + + +// WithinDuration asserts that the two times are within duration delta of each other. +// +// a.WithinDuration(time.Now(), time.Now(), 10*time.Second, "The difference should not be more than 10s") +// +// Returns whether the assertion was successful (true) or not (false). +func (a *Assertions) WithinDuration(expected time.Time, actual time.Time, delta time.Duration, msgAndArgs ...interface{}) { + WithinDuration(a.t, expected, actual, delta, msgAndArgs...) +} + + +// Zero asserts that i is the zero value for its type and returns the truth. +func (a *Assertions) Zero(i interface{}, msgAndArgs ...interface{}) { + Zero(a.t, i, msgAndArgs...) +} diff --git a/vendor/github.com/stretchr/testify/require/requirements.go b/vendor/github.com/stretchr/testify/require/requirements.go new file mode 100644 index 000000000000..41147562d862 --- /dev/null +++ b/vendor/github.com/stretchr/testify/require/requirements.go @@ -0,0 +1,9 @@ +package require + +// TestingT is an interface wrapper around *testing.T +type TestingT interface { + Errorf(format string, args ...interface{}) + FailNow() +} + +//go:generate go run ../_codegen/main.go -output-package=require -template=require.go.tmpl