oci/cas/dir: Load blob URI from oci-layout

cmd/umoci/gc.go

@@ -18,10 +18,13 @@
 package main
 
 import (
+	"regexp"
+
 	"github.com/openSUSE/umoci/oci/cas/dir"
 	"github.com/openSUSE/umoci/oci/casext"
 	"github.com/pkg/errors"
 	"github.com/urfave/cli"
+	casDir "github.com/wking/casengine/dir"
 	"golang.org/x/net/context"
 )
 
@@ -35,6 +38,12 @@ Where "<image-path>" is the path to the OCI image.
 This command will do a mark-and-sweep garbage collection of the provided OCI
 image, only retaining blobs which can be reached by a descriptor path from the
 root set of references. All other blobs will be removed.`,
+	Flags: []cli.Flag{
+		cli.StringFlag{
+			Name:  "digest-regexp",
+			Usage: "regular expression for calculating the digest from a filesystem path.  This is required if your oci-layout declares an oci-cas-template-v1 CAS engine (e.g. via 'umoci init --blob-uri ...')",
+		},
+	},
 
 	// create modifies an image layout.
 	Category: "layout",
@@ -52,8 +61,22 @@ root set of references. All other blobs will be removed.`,
 func gc(ctx *cli.Context) error {
 	imagePath := ctx.App.Metadata["--image-path"].(string)
 
+	var getDigest casDir.GetDigest
+	if ctx.IsSet("digest-regexp") {
+		getDigestRegexp, err := regexp.Compile(ctx.String("digest-regexp"))
+		if err != nil {
+			return errors.Wrap(err, "compile digest-regexp")
+		}
+
+		regexpGetDigest := &casDir.RegexpGetDigest{
+			Regexp: getDigestRegexp,
+		}
+
+		getDigest = regexpGetDigest.GetDigest
+	}
+
 	// Get a reference to the CAS.
-	engine, err := dir.Open(imagePath)
+	engine, err := dir.OpenWithDigestLister(imagePath, getDigest)
 	if err != nil {
 		return errors.Wrap(err, "open CAS")
 	}

cmd/umoci/init.go

@@ -38,6 +38,13 @@ The new OCI image does not contain any references or blobs, but those can be
 created through the use of umoci-new(1), umoci-tag(1) and other similar
 commands.`,
 
+	Flags: []cli.Flag{
+		cli.StringFlag{
+			Name:  "blob-uri",
+			Usage: "URI Template for storing blobs, interpreted with the image path as the base URI.  Defaults to blobs/{algorithm}/{encoded}",
+		},
+	},
+
 	// create modifies an image layout.
 	Category: "layout",
 
@@ -54,7 +61,7 @@ func initLayout(ctx *cli.Context) error {
 		return errors.Wrap(err, "image layout creation")
 	}
 
-	if err := dir.Create(imagePath); err != nil {
+	if err := dir.Create(imagePath, ctx.String("blob-uri")); err != nil {
 		return errors.Wrap(err, "image layout creation")
 	}
 

doc/man/umoci-gc.1.md

@@ -7,6 +7,7 @@ umoci gc - Garbage collects all unreferenced OCI image blobs
 # SYNOPSIS
 **umoci gc**
 **--layout**=*image*
+[**--digest-regexp**=*regexp*]
 
 # DESCRIPTION
 Conduct a mark-and-sweep garbage collection of the provided OCI image, only
@@ -20,6 +21,18 @@ The global options are defined in **umoci**(1).
   The OCI image layout to be garbage collected. *image* must be a path to a
   valid OCI image.
 
+**--digest-regexp**=*regexp*
+  A regular expression for calculating the digest from a filesystem
+  path.  This is required if your oci-layout declares an
+  `oci-cas-template-v1` CAS engine.  For example, if you created the
+  image with:
+
+    umoci init --blob-uri file:///path/to/my/blobs/{algorithm}/{encoded:2}/{encoded}
+
+  Then you should set *regexp* to:
+
+    ^.*/(?P<algorithm>[a-z0-9+._-]+)/[a-zA-Z0-9=_-]{1,2}/(?P<encoded>[a-zA-Z0-9=_-]{1,})$
+
 # EXAMPLE
 
 The following deletes a tag from an OCI image and clean conducts a garbage

doc/man/umoci-init.1.md

@@ -7,6 +7,7 @@ umoci init - Create a new OCI image layout
 # SYNOPSIS
 **umoci init**
 **--layout**=*image*
+[**--blob-uri**=*template*
 
 # DESCRIPTION
 Creates a new OCI image layout. The new OCI image does not contain any new
@@ -21,6 +22,11 @@ The global options are defined in **umoci**(1).
   The path where the OCI image layout will be created. The path must not exist
   already or **umoci-init**(1) will return an error.
 
+**--blob-uri**=*template*
+  The URI Template for retrieving digests.  Relative URIs will be
+  resolved with the image path as the base URI.  For more details,
+  see the [OCI CAS Template Protocol][cas-template].
+
 # EXAMPLE
 
 The following creates a brand new OCI image layout and then creates a blank tag
@@ -33,3 +39,5 @@ for further manipulation with **umoci-repack**(1) and **umoci-config**(1).
 
 # SEE ALSO
 **umoci**(1), **umoci-new**(1)
+
+[cas-template]: https://github.com/xiekeyang/oci-discovery/blob/0be7eae246ae9a975a76ca209c045043f0793572/cas-template.md

hack/vendor.sh

@@ -131,8 +131,11 @@ clone github.com/pkg/errors v0.8.0
 clone github.com/apex/log afb2e76037a5f36542c77e88ef8aef9f469b09f8
 clone github.com/urfave/cli v1.20.0
 clone github.com/cyphar/filepath-securejoin v0.2.1
-clone github.com/vbatts/go-mtree v0.4.1
-clone github.com/Sirupsen/logrus v1.0.3
+clone github.com/jtacoma/uritemplates v1.0.0
+clone github.com/vbatts/go-mtree 005af4d18f8ab74174ce23565be732a3101cf316
+clone github.com/sirupsen/logrus v1.0.3
+clone github.com/wking/casengine 3ed08888a9365a2753ab8b809b7efb286566fe8d
+clone github.com/xiekeyang/oci-discovery 17aaa9ee7538d1db09b5f142ed319e06dee7407e
 clone golang.org/x/net 45e771701b814666a7eb299e6c7a57d0b1799e91 https://github.com/golang/net
 # Used purely for testing.
 clone github.com/mohae/deepcopy 491d3605edfb866af34a48075bd4355ac1bf46ca

mutate/mutate_test.go

@@ -47,7 +47,7 @@ const (
 
 func setup(t *testing.T, dir string) (cas.Engine, ispec.Descriptor) {
 	dir = filepath.Join(dir, "image")
-	if err := casdir.Create(dir); err != nil {
+	if err := casdir.Create(dir, ""); err != nil {
 		t.Fatal(err)
 	}
 

oci/cas/cas.go

@@ -28,6 +28,7 @@ import (
 
 	"github.com/opencontainers/go-digest"
 	ispec "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/wking/casengine"
 	"golang.org/x/net/context"
 )
 
@@ -63,13 +64,24 @@ var (
 // Engine is an interface that provides methods for accessing and modifying an
 // OCI image, namely allowing access to reference descriptors and blobs.
 type Engine interface {
+	// CAS returns the casengine.Engine backing this engine.
+	CAS() (casEngine casengine.Engine)
+
+	// DigestListerEngine returns the casengine.DigestListerEngine
+	// backing this engine, or nil if no such engine exists.
+	DigestListerEngine() (casEngine casengine.DigestListerEngine)
+
 	// PutBlob adds a new blob to the image. This is idempotent; a nil error
 	// means that "the content is stored at DIGEST" without implying "because
 	// of this PutBlob() call".
+	//
+	// Deprecated: Use CAS().Put instead.
 	PutBlob(ctx context.Context, reader io.Reader) (digest digest.Digest, size int64, err error)
 
 	// GetBlob returns a reader for retrieving a blob from the image, which the
 	// caller must Close(). Returns ErrNotExist if the digest is not found.
+	//
+	// Deprecated: Use CAS().Get instead.
 	GetBlob(ctx context.Context, digest digest.Digest) (reader io.ReadCloser, err error)
 
 	// PutIndex sets the index of the OCI image to the given index, replacing
@@ -92,9 +104,13 @@ type Engine interface {
 	// DeleteBlob removes a blob from the image. This is idempotent; a nil
 	// error means "the content is not in the store" without implying "because
 	// of this DeleteBlob() call".
+	//
+	// Deprecated: Use CAS().Delete instead.
 	DeleteBlob(ctx context.Context, digest digest.Digest) (err error)
 
 	// ListBlobs returns the set of blob digests stored in the image.
+	//
+	// Deprecated: Use DigestListerEngine().Digests instead.
 	ListBlobs(ctx context.Context) (digests []digest.Digest, err error)
 
 	// Clean executes a garbage collection of any non-blob garbage in the store

oci/cas/dir/cas_test.go

@@ -19,14 +19,18 @@ package dir
 
 import (
 	"bytes"
+	"fmt"
 	"io"
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"regexp"
 	"testing"
 
 	"github.com/openSUSE/umoci/oci/cas"
+	"github.com/opencontainers/go-digest"
 	"github.com/pkg/errors"
+	"github.com/wking/casengine/dir"
 	"golang.org/x/net/context"
 )
 
@@ -43,7 +47,8 @@ func TestCreateLayout(t *testing.T) {
 	defer os.RemoveAll(root)
 
 	image := filepath.Join(root, "image")
-	if err := Create(image); err != nil {
+
+	if err := Create(image, ""); err != nil {
 		t.Fatalf("unexpected error creating image: %+v", err)
 	}
 
@@ -66,7 +71,7 @@ func TestCreateLayout(t *testing.T) {
 	}
 
 	// We should get an error if we try to create a new image atop an old one.
-	if err := Create(image); err == nil {
+	if err := Create(image, ""); err == nil {
 		t.Errorf("expected to get a cowardly no-clobber error!")
 	}
 }
@@ -81,7 +86,7 @@ func TestEngineBlob(t *testing.T) {
 	defer os.RemoveAll(root)
 
 	image := filepath.Join(root, "image")
-	if err := Create(image); err != nil {
+	if err := Create(image, ""); err != nil {
 		t.Fatalf("unexpected error creating image: %+v", err)
 	}
 
@@ -214,7 +219,7 @@ func TestEngineValidate(t *testing.T) {
 	if err := os.Remove(image); err != nil {
 		t.Fatal(err)
 	}
-	if err := Create(image); err != nil {
+	if err := Create(image, ""); err != nil {
 		t.Fatalf("unexpected error creating image: %+v", err)
 	}
 	if err := os.RemoveAll(filepath.Join(image, blobDirectory)); err != nil {
@@ -234,7 +239,7 @@ func TestEngineValidate(t *testing.T) {
 	if err := os.Remove(image); err != nil {
 		t.Fatal(err)
 	}
-	if err := Create(image); err != nil {
+	if err := Create(image, ""); err != nil {
 		t.Fatalf("unexpected error creating image: %+v", err)
 	}
 	if err := os.RemoveAll(filepath.Join(image, blobDirectory)); err != nil {
@@ -257,7 +262,7 @@ func TestEngineValidate(t *testing.T) {
 	if err := os.Remove(image); err != nil {
 		t.Fatal(err)
 	}
-	if err := Create(image); err != nil {
+	if err := Create(image, ""); err != nil {
 		t.Fatalf("unexpected error creating image: %+v", err)
 	}
 	if err := os.RemoveAll(filepath.Join(image, indexFile)); err != nil {
@@ -277,7 +282,7 @@ func TestEngineValidate(t *testing.T) {
 	if err := os.Remove(image); err != nil {
 		t.Fatal(err)
 	}
-	if err := Create(image); err != nil {
+	if err := Create(image, ""); err != nil {
 		t.Fatalf("unexpected error creating image: %+v", err)
 	}
 	if err := os.RemoveAll(filepath.Join(image, indexFile)); err != nil {
@@ -300,3 +305,73 @@ func TestEngineValidate(t *testing.T) {
 		engine.Close()
 	}
 }
+
+func TestEngineURITemplate(t *testing.T) {
+	ctx := context.Background()
+
+	root, err := ioutil.TempDir("", "umoci-TestEngineURITemplate")
+	if err != nil {
+		t.Fatal(err)
+	}
+	//defer os.RemoveAll(root)
+
+	image := filepath.Join(root, "image")
+
+	if filepath.Separator != '/' {
+		t.Fatalf("CAS URI Template initialization is not implemented for filepath.Separator %q", filepath.Separator)
+	}
+
+	if err := Create(image, fmt.Sprintf("file://%s/blobs/{algorithm}/{encoded:2}/{encoded}", root)); err != nil {
+		t.Fatalf("unexpected error creating image: %+v", err)
+	}
+
+	getDigestRegexp, err := regexp.Compile(`^.*/blobs/(?P<algorithm>[a-z0-9+._-]+)/[a-zA-Z0-9=_-]{1,2}/(?P<encoded>[a-zA-Z0-9=_-]{1,})$`)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	getDigest := &dir.RegexpGetDigest{
+		Regexp: getDigestRegexp,
+	}
+
+	engine, err := OpenWithDigestLister(image, getDigest.GetDigest)
+	if err != nil {
+		t.Fatalf("unexpected error opening image: %+v", err)
+	}
+	defer engine.Close()
+
+	bytesIn := []byte("Hello, World!")
+	dig, err := engine.CAS().Put(ctx, digest.SHA256, bytes.NewReader(bytesIn))
+	if err != nil {
+		t.Errorf("Put: unexpected error: %+v", err)
+	}
+
+	reader, err := engine.CAS().Get(ctx, dig)
+	if err != nil {
+		t.Errorf("Get: unexpected error: %+v", err)
+	}
+	defer reader.Close()
+
+	gotBytes, err := ioutil.ReadAll(reader)
+	if err != nil {
+		t.Errorf("Get: failed to ReadAll: %+v", err)
+	}
+	if !bytes.Equal(bytesIn, gotBytes) {
+		t.Errorf("Get: bytes did not match: expected=%s got=%s", string(bytesIn), string(gotBytes))
+	}
+
+	path := filepath.Join(root, "blobs", digest.SHA256.String(), "df", "dffd6021bb2bd5b0af676290809ec3a53191dd81c7f70a4b28688a362182986f")
+	reader, err = os.Open(path)
+	if err != nil {
+		t.Error(err)
+	}
+	defer reader.Close()
+
+	gotBytes, err = ioutil.ReadAll(reader)
+	if err != nil {
+		t.Errorf("Open: failed to ReadAll: %+v", err)
+	}
+	if !bytes.Equal(bytesIn, gotBytes) {
+		t.Errorf("Open: bytes did not match: expected=%s got=%s", string(bytesIn), string(gotBytes))
+	}
+}