diff --git a/content/oci/oci.go b/content/oci/oci.go index 27afde16..5d4699a1 100644 --- a/content/oci/oci.go +++ b/content/oci/oci.go @@ -14,7 +14,7 @@ limitations under the License. */ // Package oci provides access to an OCI content store. -// Reference: https://github.com/opencontainers/image-spec/blob/v1.1.0-rc4/image-layout.md +// Reference: https://github.com/opencontainers/image-spec/blob/v1.1.0-rc5/image-layout.md package oci import ( @@ -40,12 +40,14 @@ import ( // Store implements `oras.Target`, and represents a content store // based on file system with the OCI-Image layout. -// Reference: https://github.com/opencontainers/image-spec/blob/v1.1.0-rc4/image-layout.md +// Reference: https://github.com/opencontainers/image-spec/blob/v1.1.0-rc5/image-layout.md type Store struct { // AutoSaveIndex controls if the OCI store will automatically save the index - // file on each Tag() call. - // - If AutoSaveIndex is set to true, the OCI store will automatically call - // this method on each Tag() call. + // file when needed. + // - If AutoSaveIndex is set to true, the OCI store will automatically save + // the changes to `index.json` when + // 1. pushing a manifest + // 2. calling Tag() or Delete() // - If AutoSaveIndex is set to false, it's the caller's responsibility // to manually call SaveIndex() when needed. // - Default value: true. @@ -53,11 +55,16 @@ type Store struct { root string indexPath string index *ocispec.Index - indexLock sync.Mutex - - storage content.Storage - tagResolver *resolver.Memory - graph *graph.Memory + storage *Storage + tagResolver *resolver.Memory + graph *graph.Memory + + // sync ensures that most operations can be done concurrently, while Delete + // has the exclusive access to Store if a delete operation is underway. Operations + // such as Fetch, Push use sync.RLock(), while Delete uses sync.Lock(). + sync sync.RWMutex + // indexLock ensures that only one go-routine is writing to the index. + indexLock sync.Mutex } // New creates a new OCI store with context.Background(). @@ -98,13 +105,21 @@ func NewWithContext(ctx context.Context, root string) (*Store, error) { return store, nil } -// Fetch fetches the content identified by the descriptor. +// Fetch fetches the content identified by the descriptor. It returns an io.ReadCloser. +// It's recommended to close the io.ReadCloser before a Delete operation, otherwise +// Delete may fail (for example on NTFS file systems). func (s *Store) Fetch(ctx context.Context, target ocispec.Descriptor) (io.ReadCloser, error) { + s.sync.RLock() + defer s.sync.RUnlock() + return s.storage.Fetch(ctx, target) } // Push pushes the content, matching the expected descriptor. func (s *Store) Push(ctx context.Context, expected ocispec.Descriptor, reader io.Reader) error { + s.sync.RLock() + defer s.sync.RUnlock() + if err := s.storage.Push(ctx, expected, reader); err != nil { return err } @@ -120,13 +135,46 @@ func (s *Store) Push(ctx context.Context, expected ocispec.Descriptor, reader io // Exists returns true if the described content exists. func (s *Store) Exists(ctx context.Context, target ocispec.Descriptor) (bool, error) { + s.sync.RLock() + defer s.sync.RUnlock() + return s.storage.Exists(ctx, target) } +// Delete deletes the content matching the descriptor from the store. Delete may +// fail on certain systems (i.e. NTFS), if there is a process (i.e. an unclosed +// Reader) using target. +func (s *Store) Delete(ctx context.Context, target ocispec.Descriptor) error { + s.sync.Lock() + defer s.sync.Unlock() + + resolvers := s.tagResolver.Map() + untagged := false + for reference, desc := range resolvers { + if content.Equal(desc, target) { + s.tagResolver.Untag(reference) + untagged = true + } + } + if err := s.graph.Remove(ctx, target); err != nil { + return err + } + if untagged && s.AutoSaveIndex { + err := s.saveIndex() + if err != nil { + return err + } + } + return s.storage.Delete(ctx, target) +} + // Tag tags a descriptor with a reference string. // reference should be a valid tag (e.g. "latest"). -// Reference: https://github.com/opencontainers/image-spec/blob/v1.1.0-rc4/image-layout.md#indexjson-file +// Reference: https://github.com/opencontainers/image-spec/blob/v1.1.0-rc5/image-layout.md#indexjson-file func (s *Store) Tag(ctx context.Context, desc ocispec.Descriptor, reference string) error { + s.sync.RLock() + defer s.sync.RUnlock() + if err := validateReference(reference); err != nil { return err } @@ -155,7 +203,7 @@ func (s *Store) tag(ctx context.Context, desc ocispec.Descriptor, reference stri return err } if s.AutoSaveIndex { - return s.SaveIndex() + return s.saveIndex() } return nil } @@ -166,6 +214,9 @@ func (s *Store) tag(ctx context.Context, desc ocispec.Descriptor, reference stri // digest the returned descriptor will be a plain descriptor (containing only // the digest, media type and size). func (s *Store) Resolve(ctx context.Context, reference string) (ocispec.Descriptor, error) { + s.sync.RLock() + defer s.sync.RUnlock() + if reference == "" { return ocispec.Descriptor{}, errdef.ErrMissingReference } @@ -191,6 +242,9 @@ func (s *Store) Resolve(ctx context.Context, reference string) (ocispec.Descript // Predecessors returns nil without error if the node does not exists in the // store. func (s *Store) Predecessors(ctx context.Context, node ocispec.Descriptor) ([]ocispec.Descriptor, error) { + s.sync.RLock() + defer s.sync.RUnlock() + return s.graph.Predecessors(ctx, node) } @@ -202,6 +256,9 @@ func (s *Store) Predecessors(ctx context.Context, node ocispec.Descriptor) ([]oc // // See also `Tags()` in the package `registry`. func (s *Store) Tags(ctx context.Context, last string, fn func(tags []string) error) error { + s.sync.RLock() + defer s.sync.RUnlock() + return listTags(ctx, s.tagResolver, last, fn) } @@ -263,10 +320,18 @@ func (s *Store) loadIndexFile(ctx context.Context) error { // SaveIndex writes the `index.json` file to the file system. // - If AutoSaveIndex is set to true (default value), -// the OCI store will automatically call this method on each Tag() call. +// the OCI store will automatically save the changes to `index.json` +// on Tag() and Delete() calls, and when pushing a manifest. // - If AutoSaveIndex is set to false, it's the caller's responsibility // to manually call this method when needed. func (s *Store) SaveIndex() error { + s.sync.RLock() + defer s.sync.RUnlock() + + return s.saveIndex() +} + +func (s *Store) saveIndex() error { s.indexLock.Lock() defer s.indexLock.Unlock() diff --git a/content/oci/oci_test.go b/content/oci/oci_test.go index 43e57f0a..01028b1a 100644 --- a/content/oci/oci_test.go +++ b/content/oci/oci_test.go @@ -644,7 +644,7 @@ func TestStore_DisableAutoSaveIndex(t *testing.T) { if got, want := len(s.index.Manifests), 0; got != want { t.Errorf("len(index.Manifests) = %v, want %v", got, want) } - if err := s.SaveIndex(); err != nil { + if err := s.saveIndex(); err != nil { t.Fatal("Store.SaveIndex() error =", err) } // test index file again @@ -1975,6 +1975,268 @@ func TestStore_Tags(t *testing.T) { } } +func TestStore_BasicDelete(t *testing.T) { + content := []byte("test delete") + desc := ocispec.Descriptor{ + MediaType: "test-delete", + Digest: digest.FromBytes(content), + Size: int64(len(content)), + } + ref := "latest" + + tempDir := t.TempDir() + s, err := New(tempDir) + if err != nil { + t.Fatal("NewDeletableStore() error =", err) + } + ctx := context.Background() + + err = s.Push(ctx, desc, bytes.NewReader(content)) + if err != nil { + t.Errorf("Store.Push() error = %v, wantErr %v", err, false) + } + + err = s.Tag(ctx, desc, ref) + if err != nil { + t.Errorf("error tagging descriptor error = %v, wantErr %v", err, false) + } + + exists, err := s.Exists(ctx, desc) + if err != nil { + t.Fatal("Store.Exists() error =", err) + } + if !exists { + t.Errorf("Store.Exists() = %v, want %v", exists, true) + } + + resolvedDescr, err := s.Resolve(ctx, ref) + if err != nil { + t.Errorf("error resolving descriptor error = %v, wantErr %v", err, false) + } + + if !reflect.DeepEqual(resolvedDescr, desc) { + t.Errorf("Store.Resolve() = %v, want %v", resolvedDescr, desc) + } + + err = s.Delete(ctx, desc) + if err != nil { + t.Errorf("Store.Delete() = %v, wantErr %v", err, nil) + } + + exists, err = s.Exists(ctx, desc) + if err != nil { + t.Fatal("Store.Exists() error =", err) + } + if exists { + t.Errorf("Store.Exists() = %v, want %v", exists, false) + } +} + +func TestStore_FetchAndDelete(t *testing.T) { + // create a store + tempDir := t.TempDir() + s, err := New(tempDir) + if err != nil { + t.Fatal("error =", err) + } + + // push a content + content := []byte("test delete") + desc := ocispec.Descriptor{ + MediaType: "test-delete", + Digest: digest.FromBytes(content), + Size: int64(len(content)), + } + err = s.Push(context.Background(), desc, bytes.NewReader(content)) + if err != nil { + t.Fatal("error =", err) + } + + // fetch a content + rc, err := s.Fetch(context.Background(), desc) + if err != nil { + t.Fatal("error =", err) + } + + // read and verify the content + got, err := io.ReadAll(rc) + if err != nil { + t.Fatal("error =", err) + } + if !bytes.Equal(got, content) { + t.Errorf("Store.Fetch() = %v, want %v", string(got), string(content)) + } + rc.Close() + + // delete. If rc is not closed, Delete would fail on some systems. + err = s.Delete(context.Background(), desc) + if err != nil { + t.Fatal("error =", err) + } +} + +func TestStore_PredecessorsAndDelete(t *testing.T) { + tempDir := t.TempDir() + s, err := New(tempDir) + if err != nil { + t.Fatal("New() error =", err) + } + ctx := context.Background() + + // generate test content + var blobs [][]byte + var descs []ocispec.Descriptor + appendBlob := func(mediaType string, blob []byte) { + blobs = append(blobs, blob) + descs = append(descs, ocispec.Descriptor{ + MediaType: mediaType, + Digest: digest.FromBytes(blob), + Size: int64(len(blob)), + }) + } + generateManifest := func(config ocispec.Descriptor, layers ...ocispec.Descriptor) { + manifest := ocispec.Manifest{ + Config: config, + Layers: layers, + } + manifestJSON, err := json.Marshal(manifest) + if err != nil { + t.Fatal(err) + } + appendBlob(ocispec.MediaTypeImageManifest, manifestJSON) + } + generateIndex := func(manifests ...ocispec.Descriptor) { + index := ocispec.Index{ + Manifests: manifests, + } + indexJSON, err := json.Marshal(index) + if err != nil { + t.Fatal(err) + } + appendBlob(ocispec.MediaTypeImageIndex, indexJSON) + } + + appendBlob(ocispec.MediaTypeImageConfig, []byte("config")) // Blob 0 + appendBlob(ocispec.MediaTypeImageLayer, []byte("foo")) // Blob 1 + appendBlob(ocispec.MediaTypeImageLayer, []byte("bar")) // Blob 2 + appendBlob(ocispec.MediaTypeImageLayer, []byte("hello")) // Blob 3 + generateManifest(descs[0], descs[1:3]...) // Blob 4 + generateManifest(descs[0], descs[3]) // Blob 5 + generateManifest(descs[0], descs[1:4]...) // Blob 6 + generateIndex(descs[4:6]...) // Blob 7 + generateIndex(descs[6]) // Blob 8 + + eg, egCtx := errgroup.WithContext(ctx) + for i := range blobs { + eg.Go(func(i int) func() error { + return func() error { + err := s.Push(egCtx, descs[i], bytes.NewReader(blobs[i])) + if err != nil { + return fmt.Errorf("failed to push test content to src: %d: %v", i, err) + } + return nil + } + }(i)) + } + if err := eg.Wait(); err != nil { + t.Fatal(err) + } + + // verify predecessors + wants := [][]ocispec.Descriptor{ + descs[4:7], // Blob 0 + {descs[4], descs[6]}, // Blob 1 + {descs[4], descs[6]}, // Blob 2 + {descs[5], descs[6]}, // Blob 3 + {descs[7]}, // Blob 4 + {descs[7]}, // Blob 5 + {descs[8]}, // Blob 6 + nil, // Blob 7 + nil, // Blob 8 + } + for i, want := range wants { + predecessors, err := s.Predecessors(ctx, descs[i]) + if err != nil { + t.Errorf("Store.Predecessors(%d) error = %v", i, err) + } + if !equalDescriptorSet(predecessors, want) { + t.Errorf("Store.Predecessors(%d) = %v, want %v", i, predecessors, want) + } + } + + // delete a node and verify the result + s.Delete(egCtx, descs[6]) + // verify predecessors + wants = [][]ocispec.Descriptor{ + descs[4:6], // Blob 0 + {descs[4]}, // Blob 1 + {descs[4]}, // Blob 2 + {descs[5]}, // Blob 3 + {descs[7]}, // Blob 4 + {descs[7]}, // Blob 5 + {descs[8]}, // Blob 6 + nil, // Blob 7 + nil, // Blob 8 + } + for i, want := range wants { + predecessors, err := s.Predecessors(ctx, descs[i]) + if err != nil { + t.Errorf("Store.Predecessors(%d) error = %v", i, err) + } + if !equalDescriptorSet(predecessors, want) { + t.Errorf("Store.Predecessors(%d) = %v, want %v", i, predecessors, want) + } + } + + // delete a node and verify the result + s.Delete(egCtx, descs[8]) + // verify predecessors + wants = [][]ocispec.Descriptor{ + descs[4:6], // Blob 0 + {descs[4]}, // Blob 1 + {descs[4]}, // Blob 2 + {descs[5]}, // Blob 3 + {descs[7]}, // Blob 4 + {descs[7]}, // Blob 5 + nil, // Blob 6 + nil, // Blob 7 + nil, // Blob 8 + } + for i, want := range wants { + predecessors, err := s.Predecessors(ctx, descs[i]) + if err != nil { + t.Errorf("Store.Predecessors(%d) error = %v", i, err) + } + if !equalDescriptorSet(predecessors, want) { + t.Errorf("Store.Predecessors(%d) = %v, want %v", i, predecessors, want) + } + } + + // delete a node and verify the result + s.Delete(egCtx, descs[5]) + // verify predecessors + wants = [][]ocispec.Descriptor{ + {descs[4]}, // Blob 0 + {descs[4]}, // Blob 1 + {descs[4]}, // Blob 2 + nil, // Blob 3 + {descs[7]}, // Blob 4 + {descs[7]}, // Blob 5 + nil, // Blob 6 + nil, // Blob 7 + nil, // Blob 8 + } + for i, want := range wants { + predecessors, err := s.Predecessors(ctx, descs[i]) + if err != nil { + t.Errorf("Store.Predecessors(%d) error = %v", i, err) + } + if !equalDescriptorSet(predecessors, want) { + t.Errorf("Store.Predecessors(%d) = %v, want %v", i, predecessors, want) + } + } +} + func equalDescriptorSet(actual []ocispec.Descriptor, expected []ocispec.Descriptor) bool { if len(actual) != len(expected) { return false