Merge pull request #342 from tych0/add-whiteout-option

{un,re}pack: support overlayfs whiteouts

.gitignore

@@ -1,4 +1,5 @@
 /umoci
+/umoci.static
 /umoci.cov*
 /.cache
 /release

cmd/umoci/insert.go

@@ -156,7 +156,8 @@ func insert(ctx *cli.Context) error {
 		return err
 	}
 
-	reader := layer.GenerateInsertLayer(sourcePath, targetPath, ctx.IsSet("opaque"), &meta.MapOptions)
+	packOptions := layer.RepackOptions{MapOptions: meta.MapOptions}
+	reader := layer.GenerateInsertLayer(sourcePath, targetPath, ctx.IsSet("opaque"), &packOptions)
 	defer reader.Close()
 
 	var history *ispec.History

cmd/umoci/raw-unpack.go

@@ -69,6 +69,7 @@ func rawUnpack(ctx *cli.Context) error {
 	fromName := ctx.App.Metadata["--image-tag"].(string)
 	rootfsPath := ctx.App.Metadata["rootfs"].(string)
 
+	var unpackOptions layer.UnpackOptions
 	var meta umoci.Meta
 	meta.Version = umoci.MetaVersion
 
@@ -79,7 +80,8 @@ func rawUnpack(ctx *cli.Context) error {
 		return err
 	}
 
-	meta.MapOptions.KeepDirlinks = ctx.Bool("keep-dirlinks")
+	unpackOptions.KeepDirlinks = ctx.Bool("keep-dirlinks")
+	unpackOptions.MapOptions = meta.MapOptions
 
 	// Get a reference to the CAS.
 	engine, err := dir.Open(imagePath)
@@ -126,7 +128,7 @@ func rawUnpack(ctx *cli.Context) error {
 	}
 
 	log.Warnf("unpacking rootfs ...")
-	if err := layer.UnpackRootfs(context.Background(), engineExt, rootfsPath, manifest, &meta.MapOptions, nil, ispec.Descriptor{}); err != nil {
+	if err := layer.UnpackRootfs(context.Background(), engineExt, rootfsPath, manifest, &unpackOptions); err != nil {
 		return errors.Wrap(err, "create rootfs")
 	}
 	log.Warnf("... done")

cmd/umoci/unpack.go

@@ -18,10 +18,10 @@
 package main
 
 import (
-	ispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/opencontainers/umoci"
 	"github.com/opencontainers/umoci/oci/cas/dir"
 	"github.com/opencontainers/umoci/oci/casext"
+	"github.com/opencontainers/umoci/oci/layer"
 	"github.com/pkg/errors"
 	"github.com/urfave/cli"
 )
@@ -68,6 +68,7 @@ func unpack(ctx *cli.Context) error {
 	fromName := ctx.App.Metadata["--image-tag"].(string)
 	bundlePath := ctx.App.Metadata["bundle"].(string)
 
+	var unpackOptions layer.UnpackOptions
 	var meta umoci.Meta
 	meta.Version = umoci.MetaVersion
 
@@ -77,7 +78,8 @@ func unpack(ctx *cli.Context) error {
 		return err
 	}
 
-	meta.MapOptions.KeepDirlinks = ctx.Bool("keep-dirlinks")
+	unpackOptions.KeepDirlinks = ctx.Bool("keep-dirlinks")
+	unpackOptions.MapOptions = meta.MapOptions
 
 	// Get a reference to the CAS.
 	engine, err := dir.Open(imagePath)
@@ -86,5 +88,5 @@ func unpack(ctx *cli.Context) error {
 	}
 	engineExt := casext.NewEngine(engine)
 	defer engine.Close()
-	return umoci.Unpack(engineExt, fromName, bundlePath, meta.MapOptions, nil, ispec.Descriptor{})
+	return umoci.Unpack(engineExt, fromName, bundlePath, unpackOptions)
 }

go.mod

@@ -22,6 +22,7 @@ require (
 	github.com/pkg/errors v0.9.1
 	github.com/rootless-containers/proto v0.1.0
 	github.com/sirupsen/logrus v1.6.0 // indirect
+	github.com/stretchr/testify v1.6.1
 	github.com/tj/assert v0.0.3 // indirect
 	github.com/urfave/cli v1.22.4
 	github.com/vbatts/go-mtree v0.5.0

oci/layer/generate.go

@@ -43,10 +43,10 @@ func (ids inodeDeltas) Swap(i, j int)      { ids[i], ids[j] = ids[j], ids[i] }
 // provided path (which should be the rootfs of the layer that was diffed). The
 // returned reader is for the *raw* tar data, it is the caller's responsibility
 // to gzip it.
-func GenerateLayer(path string, deltas []mtree.InodeDelta, opt *MapOptions) (io.ReadCloser, error) {
-	var mapOptions MapOptions
+func GenerateLayer(path string, deltas []mtree.InodeDelta, opt *RepackOptions) (io.ReadCloser, error) {
+	var packOptions RepackOptions
 	if opt != nil {
-		mapOptions = *opt
+		packOptions = *opt
 	}
 
 	reader, writer := io.Pipe()
@@ -61,7 +61,7 @@ func GenerateLayer(path string, deltas []mtree.InodeDelta, opt *MapOptions) (io.
 		// We can't just dump all of the file contents into a tar file. We need
 		// to emulate a proper tar generator. Luckily there aren't that many
 		// things to emulate (and we can do them all in tar.go).
-		tg := newTarGenerator(writer, mapOptions)
+		tg := newTarGenerator(writer, packOptions.MapOptions)
 
 		// Sort the delta paths.
 		// FIXME: We need to add whiteouts first, otherwise we might end up
@@ -79,6 +79,19 @@ func GenerateLayer(path string, deltas []mtree.InodeDelta, opt *MapOptions) (io.
 
 			switch delta.Type() {
 			case mtree.Modified, mtree.Extra:
+				if packOptions.TranslateOverlayWhiteouts {
+					fi, err := os.Stat(fullPath)
+					if err != nil {
+						return errors.Wrapf(err, "couldn't determine overlay whiteout for %s", fullPath)
+					}
+
+					if isOverlayWhiteout(fi) {
+						if err := tg.AddWhiteout(fullPath); err != nil {
+							return errors.Wrap(err, "generate whiteout from overlayfs")
+						}
+					}
+					continue
+				}
 				if err := tg.AddFile(name, fullPath); err != nil {
 					log.Warnf("generate layer: could not add file '%s': %s", name, err)
 					return errors.Wrap(err, "generate layer file")
@@ -105,12 +118,12 @@ func GenerateLayer(path string, deltas []mtree.InodeDelta, opt *MapOptions) (io.
 // GenerateInsertLayer generates a completely new layer from "root"to be
 // inserted into the image at "target". If "root" is an empty string then the
 // "target" will be removed via a whiteout.
-func GenerateInsertLayer(root string, target string, opaque bool, opt *MapOptions) io.ReadCloser {
+func GenerateInsertLayer(root string, target string, opaque bool, opt *RepackOptions) io.ReadCloser {
 	root = CleanPath(root)
 
-	var mapOptions MapOptions
+	var packOptions RepackOptions
 	if opt != nil {
-		mapOptions = *opt
+		packOptions = *opt
 	}
 
 	reader, writer := io.Pipe()
@@ -121,7 +134,7 @@ func GenerateInsertLayer(root string, target string, opaque bool, opt *MapOption
 			_ = writer.CloseWithError(errors.Wrap(Err, "generate layer"))
 		}()
 
-		tg := newTarGenerator(writer, mapOptions)
+		tg := newTarGenerator(writer, packOptions.MapOptions)
 
 		if opaque {
 			if err := tg.AddOpaqueWhiteout(target); err != nil {
@@ -137,6 +150,11 @@ func GenerateInsertLayer(root string, target string, opaque bool, opt *MapOption
 			}
 
 			pathInTar := path.Join(target, curPath[len(root):])
+			if packOptions.TranslateOverlayWhiteouts && isOverlayWhiteout(info) {
+				log.Debugf("converting overlayfs whiteout %s to OCI whiteout", pathInTar)
+				return tg.AddWhiteout(pathInTar)
+			}
+
 			return tg.AddFile(pathInTar, curPath)
 		})
 	}()

oci/layer/generate_linux_test.go

@@ -0,0 +1,100 @@
+// +build linux
+
+package layer
+
+import (
+	"archive/tar"
+	"io"
+	"io/ioutil"
+	"os"
+	"path"
+	"testing"
+
+	"github.com/opencontainers/umoci/pkg/fseval"
+	"github.com/stretchr/testify/assert"
+	"github.com/vbatts/go-mtree"
+	"golang.org/x/sys/unix"
+)
+
+func TestInsertLayerTranslateOverlayWhiteouts(t *testing.T) {
+	assert := assert.New(t)
+	dir, err := ioutil.TempDir("", "umoci-TestTranslateOverlayWhiteouts")
+	assert.NoError(err)
+	defer os.RemoveAll(dir)
+
+	mknodOk, err := canMknod(dir)
+	if err != nil {
+		t.Fatalf("couldn't mknod in dir: %v", err)
+	}
+
+	if !mknodOk {
+		t.Skip("skipping overlayfs test on kernel < 5.8")
+	}
+
+	testNode := path.Join(dir, "test")
+	err = unix.Mknod(testNode, unix.S_IFCHR|0666, int(unix.Mkdev(0, 0)))
+	assert.NoError(err)
+
+	packOptions := RepackOptions{TranslateOverlayWhiteouts: true}
+	reader := GenerateInsertLayer(dir, "/", false, &packOptions)
+	defer reader.Close()
+
+	tr := tar.NewReader(reader)
+	hdr, err := tr.Next()
+	assert.NoError(err)
+	assert.Equal(hdr.Name, "/")
+
+	hdr, err = tr.Next()
+	assert.NoError(err)
+
+	assert.Equal(int32(hdr.Typeflag), int32(tar.TypeReg))
+	assert.Equal(hdr.Name, whPrefix+"test")
+	_, err = tr.Next()
+	assert.Equal(err, io.EOF)
+}
+
+func TestGenerateLayerTranslateOverlayWhiteouts(t *testing.T) {
+	assert := assert.New(t)
+	dir, err := ioutil.TempDir("", "umoci-TestTranslateOverlayWhiteouts")
+	assert.NoError(err)
+	defer os.RemoveAll(dir)
+
+	mknodOk, err := canMknod(dir)
+	if err != nil {
+		t.Fatalf("couldn't mknod in dir: %v", err)
+	}
+
+	if !mknodOk {
+		t.Skip("skipping overlayfs test on kernel < 5.8")
+	}
+
+	testNode := path.Join(dir, "test")
+	err = unix.Mknod(testNode, unix.S_IFCHR|0666, int(unix.Mkdev(0, 0)))
+	assert.NoError(err)
+
+	packOptions := RepackOptions{TranslateOverlayWhiteouts: true}
+	// something reasonable
+	mtreeKeywords := []mtree.Keyword{
+		"size",
+		"type",
+		"uid",
+		"gid",
+		"mode",
+	}
+	deltas, err := mtree.Check(dir, nil, mtreeKeywords, fseval.Default)
+	assert.NoError(err)
+
+	reader, err := GenerateLayer(dir, deltas, &packOptions)
+	assert.NoError(err)
+	defer reader.Close()
+
+	tr := tar.NewReader(reader)
+
+	hdr, err := tr.Next()
+	assert.NoError(err)
+
+	assert.Equal(int32(hdr.Typeflag), int32(tar.TypeReg))
+	assert.Equal(path.Base(hdr.Name), whPrefix+"test")
+	_, err = tr.Next()
+	assert.Equal(err, io.EOF)
+}

oci/layer/generate_test.go

@@ -74,7 +74,7 @@ func TestGenerate(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	reader, err := GenerateLayer(dir, diffs, &MapOptions{})
+	reader, err := GenerateLayer(dir, diffs, &RepackOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -188,7 +188,7 @@ func TestGenerateMissingFileError(t *testing.T) {
 	}
 
 	// Generate a layer where the changed file is missing after the diff.
-	reader, err := GenerateLayer(dir, diffs, &MapOptions{})
+	reader, err := GenerateLayer(dir, diffs, &RepackOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -253,7 +253,7 @@ func TestGenerateWrongRootError(t *testing.T) {
 	}
 
 	// Generate a layer with the wrong root directory.
-	reader, err := GenerateLayer(filepath.Join(dir, "some"), diffs, &MapOptions{})
+	reader, err := GenerateLayer(filepath.Join(dir, "some"), diffs, &RepackOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}