From 40b7e6d655197d866ccc9f2ff61bc535dbd664bc Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Thu, 19 Oct 2017 22:25:45 -0700 Subject: [PATCH] oci/cas/dir: Load blob URI from oci-layout With this change, users can configure their blob storage once at init time with an optional --blob-uri. Most other commands (which do not need path -> blob conversion) can load the blob location from the oci-layout layout file (the 1.1.0 format is in flight with [1,2]). The only other user-facing change is to 'umoci gc', which gains a --digest-regexp. Folks who customized their blob URI will need to supply --digest-regexp to reverse whichever blob URI they're using. This seems like a more convenient interface to me than requiring all callers to provide the custom blob location [3]. And it is more powerful as well, allowing users to shard their blob storage [4], etc. if they feel moved to do so. [1]: https://github.com/xiekeyang/oci-discovery/issues/20 [2]: https://github.com/wking/image-spec/blob/ref-engine-discovery-layout/image-layout.md [3]: https://github.com/openSUSE/umoci/pull/190 [4]: https://github.com/opencontainers/image-spec/issues/449 Signed-off-by: W. Trevor King --- cmd/umoci/gc.go | 25 +++- cmd/umoci/init.go | 9 +- doc/man/umoci-gc.1.md | 13 ++ doc/man/umoci-init.1.md | 8 ++ hack/vendor.sh | 1 + mutate/mutate_test.go | 2 +- oci/cas/cas.go | 6 +- oci/cas/dir/cas_test.go | 89 ++++++++++++- oci/cas/dir/dir.go | 226 +++++++++++++++++++++------------ oci/cas/dir/dir_test.go | 43 ++++++- oci/casext/json_dir_test.go | 4 +- oci/casext/refname_dir_test.go | 4 +- oci/layer/unpack_test.go | 2 +- test/create.bats | 34 +++++ 14 files changed, 366 insertions(+), 100 deletions(-) mode change 100644 => 100755 test/create.bats diff --git a/cmd/umoci/gc.go b/cmd/umoci/gc.go index 45ad00905..19866372c 100644 --- a/cmd/umoci/gc.go +++ b/cmd/umoci/gc.go @@ -18,10 +18,13 @@ package main import ( + "regexp" + "github.com/openSUSE/umoci/oci/cas/dir" "github.com/openSUSE/umoci/oci/casext" "github.com/pkg/errors" "github.com/urfave/cli" + casDir "github.com/wking/casengine/dir" "golang.org/x/net/context" ) @@ -35,6 +38,12 @@ Where "" is the path to the OCI image. This command will do a mark-and-sweep garbage collection of the provided OCI image, only retaining blobs which can be reached by a descriptor path from the root set of references. All other blobs will be removed.`, + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "digest-regexp", + Usage: "regular expression for calculating the digest from a filesystem path. This is required if your oci-layout declares an oci-cas-template-v1 CAS engine (e.g. via 'umoci init --blob-uri ...')", + }, + }, // create modifies an image layout. Category: "layout", @@ -52,8 +61,22 @@ root set of references. All other blobs will be removed.`, func gc(ctx *cli.Context) error { imagePath := ctx.App.Metadata["--image-path"].(string) + var getDigest casDir.GetDigest + if ctx.IsSet("digest-regexp") { + getDigestRegexp, err := regexp.Compile(ctx.String("digest-regexp")) + if err != nil { + return errors.Wrap(err, "compile digest-regexp") + } + + regexpGetDigest := &casDir.RegexpGetDigest{ + Regexp: getDigestRegexp, + } + + getDigest = regexpGetDigest.GetDigest + } + // Get a reference to the CAS. - engine, err := dir.Open(imagePath) + engine, err := dir.OpenWithDigestLister(imagePath, getDigest) if err != nil { return errors.Wrap(err, "open CAS") } diff --git a/cmd/umoci/init.go b/cmd/umoci/init.go index f1c47e886..215e496a7 100644 --- a/cmd/umoci/init.go +++ b/cmd/umoci/init.go @@ -38,6 +38,13 @@ The new OCI image does not contain any references or blobs, but those can be created through the use of umoci-new(1), umoci-tag(1) and other similar commands.`, + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "blob-uri", + Usage: "URI Template for storing blobs, interpreted with the image path as the base URI. Defaults to blobs/{algorithm}/{encoded}", + }, + }, + // create modifies an image layout. Category: "layout", @@ -54,7 +61,7 @@ func initLayout(ctx *cli.Context) error { return errors.Wrap(err, "image layout creation") } - if err := dir.Create(imagePath); err != nil { + if err := dir.Create(imagePath, ctx.String("blob-uri")); err != nil { return errors.Wrap(err, "image layout creation") } diff --git a/doc/man/umoci-gc.1.md b/doc/man/umoci-gc.1.md index 432a4dd42..8bf067794 100644 --- a/doc/man/umoci-gc.1.md +++ b/doc/man/umoci-gc.1.md @@ -7,6 +7,7 @@ umoci gc - Garbage collects all unreferenced OCI image blobs # SYNOPSIS **umoci gc** **--layout**=*image* +[**--digest-regexp**=*regexp*] # DESCRIPTION Conduct a mark-and-sweep garbage collection of the provided OCI image, only @@ -20,6 +21,18 @@ The global options are defined in **umoci**(1). The OCI image layout to be garbage collected. *image* must be a path to a valid OCI image. +**--digest-regexp**=*regexp* + A regular expression for calculating the digest from a filesystem + path. This is required if your oci-layout declares an + `oci-cas-template-v1` CAS engine. For example, if you created the + image with: + + umoci init --blob-uri file:///path/to/my/blobs/{algorithm}/{encoded:2}/{encoded} + + Then you should set *regexp* to: + + ^.*/(?P[a-z0-9+._-]+)/[a-zA-Z0-9=_-]{1,2}/(?P[a-zA-Z0-9=_-]{1,})$ + # EXAMPLE The following deletes a tag from an OCI image and clean conducts a garbage diff --git a/doc/man/umoci-init.1.md b/doc/man/umoci-init.1.md index 15235e0ad..ae0c7d5e7 100644 --- a/doc/man/umoci-init.1.md +++ b/doc/man/umoci-init.1.md @@ -7,6 +7,7 @@ umoci init - Create a new OCI image layout # SYNOPSIS **umoci init** **--layout**=*image* +[**--blob-uri**=*template* # DESCRIPTION Creates a new OCI image layout. The new OCI image does not contain any new @@ -21,6 +22,11 @@ The global options are defined in **umoci**(1). The path where the OCI image layout will be created. The path must not exist already or **umoci-init**(1) will return an error. +**--blob-uri**=*template* + The URI Template for retrieving digests. Relative URIs will be + resolved with the image path as the base URI. For more details, + see the [OCI CAS Template Protocol][cas-template]. + # EXAMPLE The following creates a brand new OCI image layout and then creates a blank tag @@ -33,3 +39,5 @@ for further manipulation with **umoci-repack**(1) and **umoci-config**(1). # SEE ALSO **umoci**(1), **umoci-new**(1) + +[cas-template]: https://github.com/xiekeyang/oci-discovery/blob/0be7eae246ae9a975a76ca209c045043f0793572/cas-template.md diff --git a/hack/vendor.sh b/hack/vendor.sh index 0948eddba..033b8e4e9 100755 --- a/hack/vendor.sh +++ b/hack/vendor.sh @@ -135,6 +135,7 @@ clone github.com/jtacoma/uritemplates v1.0.0 clone github.com/vbatts/go-mtree 005af4d18f8ab74174ce23565be732a3101cf316 clone github.com/sirupsen/logrus v1.0.3 clone github.com/wking/casengine 3ed08888a9365a2753ab8b809b7efb286566fe8d +clone github.com/xiekeyang/oci-discovery 17aaa9ee7538d1db09b5f142ed319e06dee7407e clone golang.org/x/net 45e771701b814666a7eb299e6c7a57d0b1799e91 https://github.com/golang/net # Used purely for testing. clone github.com/mohae/deepcopy 491d3605edfb866af34a48075bd4355ac1bf46ca diff --git a/mutate/mutate_test.go b/mutate/mutate_test.go index 49f5b7424..b89d06bdc 100644 --- a/mutate/mutate_test.go +++ b/mutate/mutate_test.go @@ -47,7 +47,7 @@ const ( func setup(t *testing.T, dir string) (cas.Engine, ispec.Descriptor) { dir = filepath.Join(dir, "image") - if err := casdir.Create(dir); err != nil { + if err := casdir.Create(dir, ""); err != nil { t.Fatal(err) } diff --git a/oci/cas/cas.go b/oci/cas/cas.go index 2aaba0dfa..c4611e88b 100644 --- a/oci/cas/cas.go +++ b/oci/cas/cas.go @@ -67,6 +67,10 @@ type Engine interface { // CAS returns the casengine.Engine backing this engine. CAS() (casEngine casengine.Engine) + // DigestListerEngine returns the casengine.DigestListerEngine + // backing this engine, or nil if no such engine exists. + DigestListerEngine() (casEngine casengine.DigestListerEngine) + // PutBlob adds a new blob to the image. This is idempotent; a nil error // means that "the content is stored at DIGEST" without implying "because // of this PutBlob() call". @@ -106,7 +110,7 @@ type Engine interface { // ListBlobs returns the set of blob digests stored in the image. // - // Deprecated: Use CAS().Digests instead. + // Deprecated: Use DigestListerEngine().Digests instead. ListBlobs(ctx context.Context) (digests []digest.Digest, err error) // Clean executes a garbage collection of any non-blob garbage in the store diff --git a/oci/cas/dir/cas_test.go b/oci/cas/dir/cas_test.go index 806158c4e..22ee5cb1d 100644 --- a/oci/cas/dir/cas_test.go +++ b/oci/cas/dir/cas_test.go @@ -19,14 +19,18 @@ package dir import ( "bytes" + "fmt" "io" "io/ioutil" "os" "path/filepath" + "regexp" "testing" "github.com/openSUSE/umoci/oci/cas" + "github.com/opencontainers/go-digest" "github.com/pkg/errors" + "github.com/wking/casengine/dir" "golang.org/x/net/context" ) @@ -43,7 +47,8 @@ func TestCreateLayout(t *testing.T) { defer os.RemoveAll(root) image := filepath.Join(root, "image") - if err := Create(image); err != nil { + + if err := Create(image, ""); err != nil { t.Fatalf("unexpected error creating image: %+v", err) } @@ -66,7 +71,7 @@ func TestCreateLayout(t *testing.T) { } // We should get an error if we try to create a new image atop an old one. - if err := Create(image); err == nil { + if err := Create(image, ""); err == nil { t.Errorf("expected to get a cowardly no-clobber error!") } } @@ -81,7 +86,7 @@ func TestEngineBlob(t *testing.T) { defer os.RemoveAll(root) image := filepath.Join(root, "image") - if err := Create(image); err != nil { + if err := Create(image, ""); err != nil { t.Fatalf("unexpected error creating image: %+v", err) } @@ -214,7 +219,7 @@ func TestEngineValidate(t *testing.T) { if err := os.Remove(image); err != nil { t.Fatal(err) } - if err := Create(image); err != nil { + if err := Create(image, ""); err != nil { t.Fatalf("unexpected error creating image: %+v", err) } if err := os.RemoveAll(filepath.Join(image, blobDirectory)); err != nil { @@ -234,7 +239,7 @@ func TestEngineValidate(t *testing.T) { if err := os.Remove(image); err != nil { t.Fatal(err) } - if err := Create(image); err != nil { + if err := Create(image, ""); err != nil { t.Fatalf("unexpected error creating image: %+v", err) } if err := os.RemoveAll(filepath.Join(image, blobDirectory)); err != nil { @@ -257,7 +262,7 @@ func TestEngineValidate(t *testing.T) { if err := os.Remove(image); err != nil { t.Fatal(err) } - if err := Create(image); err != nil { + if err := Create(image, ""); err != nil { t.Fatalf("unexpected error creating image: %+v", err) } if err := os.RemoveAll(filepath.Join(image, indexFile)); err != nil { @@ -277,7 +282,7 @@ func TestEngineValidate(t *testing.T) { if err := os.Remove(image); err != nil { t.Fatal(err) } - if err := Create(image); err != nil { + if err := Create(image, ""); err != nil { t.Fatalf("unexpected error creating image: %+v", err) } if err := os.RemoveAll(filepath.Join(image, indexFile)); err != nil { @@ -300,3 +305,73 @@ func TestEngineValidate(t *testing.T) { engine.Close() } } + +func TestEngineURITemplate(t *testing.T) { + ctx := context.Background() + + root, err := ioutil.TempDir("", "umoci-TestEngineURITemplate") + if err != nil { + t.Fatal(err) + } + //defer os.RemoveAll(root) + + image := filepath.Join(root, "image") + + if filepath.Separator != '/' { + t.Fatalf("CAS URI Template initialization is not implemented for filepath.Separator %q", filepath.Separator) + } + + if err := Create(image, fmt.Sprintf("file://%s/blobs/{algorithm}/{encoded:2}/{encoded}", root)); err != nil { + t.Fatalf("unexpected error creating image: %+v", err) + } + + getDigestRegexp, err := regexp.Compile(`^.*/blobs/(?P[a-z0-9+._-]+)/[a-zA-Z0-9=_-]{1,2}/(?P[a-zA-Z0-9=_-]{1,})$`) + if err != nil { + t.Fatal(err) + } + + getDigest := &dir.RegexpGetDigest{ + Regexp: getDigestRegexp, + } + + engine, err := OpenWithDigestLister(image, getDigest.GetDigest) + if err != nil { + t.Fatalf("unexpected error opening image: %+v", err) + } + defer engine.Close() + + bytesIn := []byte("Hello, World!") + dig, err := engine.CAS().Put(ctx, digest.SHA256, bytes.NewReader(bytesIn)) + if err != nil { + t.Errorf("Put: unexpected error: %+v", err) + } + + reader, err := engine.CAS().Get(ctx, dig) + if err != nil { + t.Errorf("Get: unexpected error: %+v", err) + } + defer reader.Close() + + gotBytes, err := ioutil.ReadAll(reader) + if err != nil { + t.Errorf("Get: failed to ReadAll: %+v", err) + } + if !bytes.Equal(bytesIn, gotBytes) { + t.Errorf("Get: bytes did not match: expected=%s got=%s", string(bytesIn), string(gotBytes)) + } + + path := filepath.Join(root, "blobs", digest.SHA256.String(), "df", "dffd6021bb2bd5b0af676290809ec3a53191dd81c7f70a4b28688a362182986f") + reader, err = os.Open(path) + if err != nil { + t.Error(err) + } + defer reader.Close() + + gotBytes, err = ioutil.ReadAll(reader) + if err != nil { + t.Errorf("Open: failed to ReadAll: %+v", err) + } + if !bytes.Equal(bytesIn, gotBytes) { + t.Errorf("Open: bytes did not match: expected=%s got=%s", string(bytesIn), string(gotBytes)) + } +} diff --git a/oci/cas/dir/dir.go b/oci/cas/dir/dir.go index 574a8ffee..435d0d8b4 100644 --- a/oci/cas/dir/dir.go +++ b/oci/cas/dir/dir.go @@ -36,17 +36,12 @@ import ( "github.com/wking/casengine" "github.com/wking/casengine/counter" "github.com/wking/casengine/dir" + "github.com/xiekeyang/oci-discovery/tools/refenginediscovery" "golang.org/x/net/context" "golang.org/x/sys/unix" ) const ( - // ImageLayoutVersion is the version of the image layout we support. This - // value is *not* the same as imagespec.Version, and the meaning of this - // field is still under discussion in the spec. For now we'll just hardcode - // the value and hope for the best. - ImageLayoutVersion = "1.0.0" - // blobDirectory is the directory inside an OCI image that contains blobs. // // FIXME: if the URI Template currently hard-coded Open() changes, @@ -64,10 +59,11 @@ const ( ) type dirEngine struct { - cas casengine.DigestListerEngine - path string - temp string - tempFile *os.File + cas casengine.Engine + digestListerEngine casengine.DigestListerEngine + path string + temp string + tempFile *os.File } func (e *dirEngine) ensureTempDir() error { @@ -94,57 +90,17 @@ func (e *dirEngine) ensureTempDir() error { return nil } -// verify ensures that the image is valid. -func (e *dirEngine) validate() error { - content, err := ioutil.ReadFile(filepath.Join(e.path, layoutFile)) - if err != nil { - if os.IsNotExist(err) { - err = cas.ErrInvalid - } - return errors.Wrap(err, "read oci-layout") - } - - var ociLayout ispec.ImageLayout - if err := json.Unmarshal(content, &ociLayout); err != nil { - return errors.Wrap(err, "parse oci-layout") - } - - // XXX: Currently the meaning of this field is not adequately defined by - // the spec, nor is the "official" value determined by the spec. - if ociLayout.Version != ImageLayoutVersion { - return errors.Wrap(cas.ErrInvalid, "layout version is not supported") - } - - // Check that "blobs" and "index.json" exist in the image. - // FIXME: We also should check that blobs *only* contains a cas.BlobAlgorithm - // directory (with no subdirectories) and that refs *only* contains - // files (optionally also making sure they're all JSON descriptors). - if fi, err := os.Stat(filepath.Join(e.path, blobDirectory)); err != nil { - if os.IsNotExist(err) { - err = cas.ErrInvalid - } - return errors.Wrap(err, "check blobdir") - } else if !fi.IsDir() { - return errors.Wrap(cas.ErrInvalid, "blobdir is not a directory") - } - - if fi, err := os.Stat(filepath.Join(e.path, indexFile)); err != nil { - if os.IsNotExist(err) { - err = cas.ErrInvalid - } - return errors.Wrap(err, "check index") - } else if fi.IsDir() { - return errors.Wrap(cas.ErrInvalid, "index is a directory") - } - - return nil -} - // CAS returns the casengine.Engine backing this engine. func (e *dirEngine) CAS() (casEngine casengine.Engine) { return e.cas } +// DigestListerEngine returns the casengine.DigestListerEngine backing +// this engine. +func (e *dirEngine) DigestListerEngine() (casEngine casengine.DigestListerEngine) { + return e.digestListerEngine +} + // PutBlob adds a new blob to the image. This is idempotent; a nil error // means that "the content is stored at DIGEST" without implying "because // of this PutBlob() call". @@ -234,10 +190,14 @@ func (e *dirEngine) DeleteBlob(ctx context.Context, digest digest.Digest) error // ListBlobs returns the set of blob digests stored in the image. // -// Deprecated: Use CAS().Digests instead. +// Deprecated: Use DigestListerEngine().Digests instead. func (e *dirEngine) ListBlobs(ctx context.Context) ([]digest.Digest, error) { + if e.digestListerEngine == nil { + return nil, fmt.Errorf("cannot list blobs without a DigestListerEngine") + } + digests := []digest.Digest{} - err := e.cas.Digests(ctx, "", "", -1, 0, func(ctx context.Context, digest digest.Digest) (err error) { + err := e.digestListerEngine.Digests(ctx, "", "", -1, 0, func(ctx context.Context, digest digest.Digest) (err error) { digests = append(digests, digest) return nil }) @@ -326,12 +286,117 @@ func (e *dirEngine) Close() (err error) { return err } -// Open opens a new reference to the directory-backed OCI image referenced by -// the provided path. -func Open(path string) (cas.Engine, error) { +// Open opens a new reference to the directory-backed OCI image +// referenced by the provided path. If your image configures a custom +// blob URI, use OpenWithDigestLister instead. +func Open(path string) (engine cas.Engine, err error) { + return OpenWithDigestLister(path, nil) +} + +// OpenWithDigestLister opens a new reference to the directory-backed +// OCI image referenced by the provided path. Use this function +// instead of Open when your image configures a custom blob URI. +func OpenWithDigestLister(path string, getDigest dir.GetDigest) (engine cas.Engine, err error) { ctx := context.Background() + + configBytes, err := ioutil.ReadFile(filepath.Join(path, layoutFile)) + if err != nil { + if os.IsNotExist(err) { + err = cas.ErrInvalid + } + return nil, errors.Wrap(err, "read oci-layout") + } + + var ociLayout ispec.ImageLayout + if err := json.Unmarshal(configBytes, &ociLayout); err != nil { + return nil, errors.Wrap(err, "parse oci-layout") + } + uri := "blobs/{algorithm}/{encoded}" + // XXX: Currently the meaning of this field is not adequately defined by + // the spec, nor is the "official" value determined by the spec. + switch ociLayout.Version { + case "1.0.0": // nothing to configure here + case "1.1.0": + var engines refenginediscovery.Engines + if err := json.Unmarshal(configBytes, &engines); err != nil { + return nil, errors.Wrap(err, "parse oci-layout") + } + for _, config := range engines.CASEngines { + if config.Protocol == "oci-cas-template-v1" { + uriInterface, ok := config.Data["uri"] + if !ok { + return nil, fmt.Errorf("CAS-template config missing required 'uri' property: %v", config.Data) + } + + uri, ok = uriInterface.(string) + if !ok { + return nil, fmt.Errorf("CAS-template config 'uri' is not a string: %v", uriInterface) + } + + break + } + } + default: + return nil, errors.Wrap(cas.ErrInvalid, fmt.Sprintf("layout version %s is not supported", ociLayout.Version)) + } + + if uri == "blobs/{algorithm}/{encoded}" { + getDigest, err = defaultGetDigest() + if err != nil { + return nil, err + } + } + + var casEngine casengine.Engine + var digestListerEngine casengine.DigestListerEngine + if getDigest == nil { + casEngine, err = dir.NewEngine(ctx, path, uri) + if err != nil { + return nil, errors.Wrap(err, "initialize CAS engine") + } + } else { + digestListerEngine, err = dir.NewDigestListerEngine(ctx, path, uri, getDigest) + if err != nil { + return nil, errors.Wrap(err, "initialize CAS engine") + } + casEngine = digestListerEngine + } + defer func() { + if err != nil { + casEngine.Close(ctx) + } + }() + + // Check that "blobs" and "index.json" exist in the image. + if fi, err := os.Stat(filepath.Join(path, blobDirectory)); err != nil { + if os.IsNotExist(err) { + err = cas.ErrInvalid + } + return nil, errors.Wrap(err, "check blobdir") + } else if !fi.IsDir() { + return nil, errors.Wrap(cas.ErrInvalid, "blobdir is not a directory") + } + + if fi, err := os.Stat(filepath.Join(path, indexFile)); err != nil { + if os.IsNotExist(err) { + err = cas.ErrInvalid + } + return nil, errors.Wrap(err, "check index") + } else if fi.IsDir() { + return nil, errors.Wrap(cas.ErrInvalid, "index is a directory") + } + + return &dirEngine{ + cas: casEngine, + digestListerEngine: digestListerEngine, + path: path, + temp: "", + }, nil +} + +func defaultGetDigest() (getDigest dir.GetDigest, err error) { pattern := `^blobs/(?P[a-z0-9+._-]+)/(?P[a-zA-Z0-9=_-]{1,})$` if filepath.Separator != '/' { if filepath.Separator == '\\' { @@ -346,32 +411,17 @@ func Open(path string) (cas.Engine, error) { return nil, errors.Wrap(err, "get-digest regexp") } - getDigest := &dir.RegexpGetDigest{ + regexpGetDigest := &dir.RegexpGetDigest{ Regexp: getDigestRegexp, } - casEngine, err := dir.NewDigestListerEngine(ctx, path, uri, getDigest.GetDigest) - if err != nil { - return nil, errors.Wrap(err, "initialize CAS engine") - } - - engine := &dirEngine{ - cas: casEngine, - path: path, - temp: "", - } - - if err := engine.validate(); err != nil { - return nil, errors.Wrap(err, "validate") - } - - return engine, nil + return regexpGetDigest.GetDigest, nil } // Create creates a new OCI image layout at the given path. If the path already // exists, os.ErrExist is returned. However, all of the parent components of // the path will be created if necessary. -func Create(path string) error { +func Create(path string, uri string) error { // We need to fail if path already exists, but we first create all of the // parent paths. dir := filepath.Dir(path) @@ -413,8 +463,22 @@ func Create(path string) error { } defer layoutFh.Close() - ociLayout := ispec.ImageLayout{ - Version: ImageLayoutVersion, + var ociLayout interface{} + switch uri { + case "": + ociLayout = ispec.ImageLayout{ + Version: "1.0.0", + } + default: + ociLayout = map[string]interface{}{ + "imageLayoutVersion": "1.1.0", + "casEngines": []map[string]interface{}{ + map[string]interface{}{ + "protocol": "oci-cas-template-v1", + "uri": uri, + }, + }, + } } if err := json.NewEncoder(layoutFh).Encode(ociLayout); err != nil { return errors.Wrap(err, "encode oci-layout") diff --git a/oci/cas/dir/dir_test.go b/oci/cas/dir/dir_test.go index 2413ee8ca..006896dfa 100644 --- a/oci/cas/dir/dir_test.go +++ b/oci/cas/dir/dir_test.go @@ -23,6 +23,7 @@ import ( "io/ioutil" "os" "path/filepath" + "regexp" "testing" "github.com/openSUSE/umoci/oci/cas" @@ -79,7 +80,7 @@ func TestCreateLayoutReadonly(t *testing.T) { defer os.RemoveAll(root) image := filepath.Join(root, "image") - if err := Create(image); err != nil { + if err := Create(image, ""); err != nil { t.Fatalf("unexpected error creating image: %+v", err) } @@ -116,7 +117,7 @@ func TestEngineBlobReadonly(t *testing.T) { defer os.RemoveAll(root) image := filepath.Join(root, "image") - if err := Create(image); err != nil { + if err := Create(image, ""); err != nil { t.Fatalf("unexpected error creating image: %+v", err) } @@ -204,7 +205,7 @@ func TestEngineGCLocking(t *testing.T) { defer os.RemoveAll(root) image := filepath.Join(root, "image") - if err := Create(image); err != nil { + if err := Create(image, ""); err != nil { t.Fatalf("unexpected error creating image: %+v", err) } @@ -285,3 +286,39 @@ func TestEngineGCLocking(t *testing.T) { } } } + +// Check for error when getDigest is needed but not provided. +func TestEngineMissingGetDigest(t *testing.T) { + ctx := context.Background() + + root, err := ioutil.TempDir("", "umoci-TestCreateLayoutReadonly") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(root) + + image := filepath.Join(root, "image") + if err := Create(image, "file:///some/where/{blob}/{algorithm:2}/{algorithm}"); err != nil { + t.Fatalf("unexpected error creating image: %+v", err) + } + + engine, err := OpenWithDigestLister(image, nil) + if err != nil { + t.Fatal(err) + } + + _, err = engine.ListBlobs(ctx) + if err == nil { + t.Fatal("open did not raise an error") + } + matched, err2 := regexp.MatchString( + `^cannot list blobs without a DigestListerEngine$`, + err.Error(), + ) + if err2 != nil { + t.Fatal(err2) + } + if !matched { + t.Fatal("open did not raise the expected error: %s", err) + } +} diff --git a/oci/casext/json_dir_test.go b/oci/casext/json_dir_test.go index 3ec2072fd..a1fc16de1 100644 --- a/oci/casext/json_dir_test.go +++ b/oci/casext/json_dir_test.go @@ -40,7 +40,7 @@ func TestEngineBlobJSON(t *testing.T) { defer os.RemoveAll(root) image := filepath.Join(root, "image") - if err := dir.Create(image); err != nil { + if err := dir.Create(image, ""); err != nil { t.Fatalf("unexpected error creating image: %+v", err) } @@ -124,7 +124,7 @@ func TestEngineBlobJSONReadonly(t *testing.T) { defer os.RemoveAll(root) image := filepath.Join(root, "image") - if err := dir.Create(image); err != nil { + if err := dir.Create(image, ""); err != nil { t.Fatalf("unexpected error creating image: %+v", err) } diff --git a/oci/casext/refname_dir_test.go b/oci/casext/refname_dir_test.go index d1815e866..4e79cff0c 100644 --- a/oci/casext/refname_dir_test.go +++ b/oci/casext/refname_dir_test.go @@ -246,7 +246,7 @@ func TestEngineReference(t *testing.T) { defer os.RemoveAll(root) image := filepath.Join(root, "image") - if err := dir.Create(image); err != nil { + if err := dir.Create(image, ""); err != nil { t.Fatalf("unexpected error creating image: %+v", err) } @@ -310,7 +310,7 @@ func TestEngineReferenceReadonly(t *testing.T) { defer os.RemoveAll(root) image := filepath.Join(root, "image") - if err := dir.Create(image); err != nil { + if err := dir.Create(image, ""); err != nil { t.Fatalf("unexpected error creating image: %+v", err) } diff --git a/oci/layer/unpack_test.go b/oci/layer/unpack_test.go index 51148ca98..13170b3e4 100644 --- a/oci/layer/unpack_test.go +++ b/oci/layer/unpack_test.go @@ -88,7 +88,7 @@ yRAbACGEEEIIIYQQQgghhBBCCKEr+wTE0sQyACgAAA==`, // Create our image. image := filepath.Join(root, "image") - if err := dir.Create(image); err != nil { + if err := dir.Create(image, ""); err != nil { t.Fatal(err) } engine, err := dir.Open(image) diff --git a/test/create.bats b/test/create.bats old mode 100644 new mode 100755 index 55b32a1a7..85c5cc565 --- a/test/create.bats +++ b/test/create.bats @@ -65,6 +65,40 @@ function teardown() { image-verify "$NEWIMAGE" } +@test "umoci init --blob-uri file://... --layout ..." { + # Setup up $NEWIMAGE. + NEWIMAGE="$(setup_tmpdir)" + rm -rf "$NEWIMAGE" + + # Create a separate directory for CAS blobs + CAS="$(setup_tmpdir)" + + # Create a new image with no tags. + umoci init --blob-uri "file://${CAS}" --layout "$NEWIMAGE" + [ "$status" -eq 0 ] + image-verify "$NEWIMAGE" + + # Make sure that there are no references or blobs. + sane_run find "$NEWIMAGE/blobs" -type f + [ "$status" -eq 0 ] + [ "${#lines[@]}" -eq 0 ] + # Note that this is _very_ dodgy at the moment because of how complicated + # the reference handling is now. + # XXX: Make sure to update this for 1.0.0-rc6 where the refname changed. + sane_run jq -SMr '.manifests[]? | .annotations["org.opencontainers.ref.name"] | strings' "$NEWIMAGE/index.json" + [ "$status" -eq 0 ] + [ "${#lines[@]}" -eq 0 ] + + # Make sure that the required files exist. + [ -f "$NEWIMAGE/oci-layout" ] + [ -d "$NEWIMAGE/blobs" ] + [ -f "$NEWIMAGE/index.json" ] + + # FIXME: check that oci-layout contains the expected casEngines. + + image-verify "$NEWIMAGE" +} + @test "umoci new [missing args]" { umoci new [ "$status" -ne 0 ]