Add Unpack Methods

Uncompressing with tar and --xattrs is too much depending on the system
configuration which is being run on. For example, it seems with certain
versions of gnutar it's not possible to extract cap bits, and bsdtar should be
used instead.

With this change we glob methods used by moby, containerd and umoci to unpack layers
and thus removing the tar dependency when unpacking docker image layers.

api/download.go

@@ -15,16 +15,29 @@ import (
 
 const defaultRegistryBase = "https://registry-1.docker.io"
 
-func DownloadImage(sourceImage, output, registryBase string) error {
+type DownloadOpts struct {
+	RegistryBase string
+	KeepLayers   bool
+}
+
+func DownloadImage(sourceImage, output string, opts *DownloadOpts) error {
 
-	if registryBase == "" {
-		registryBase = defaultRegistryBase
+	if opts.RegistryBase == "" {
+		opts.RegistryBase = defaultRegistryBase
 	}
 
 	var TempDir = os.Getenv("TEMP_LAYER_FOLDER")
-	if len(TempDir) == 0 {
+	if TempDir == "" {
 		TempDir = "layers"
 	}
+	err := os.MkdirAll(TempDir, os.ModePerm)
+	if err != nil {
+		return err
+	}
+	if opts.KeepLayers == false {
+		defer os.RemoveAll(TempDir)
+	}
+
 	if sourceImage != "" && strings.Contains(sourceImage, ":") {
 		parts := strings.Split(sourceImage, ":")
 		if parts[0] == "" || parts[1] == "" {
@@ -44,7 +57,7 @@ func DownloadImage(sourceImage, output, registryBase string) error {
 	os.MkdirAll(output, os.ModePerm)
 	username := "" // anonymous
 	password := "" // anonymous
-	hub, err := registry.New(registryBase, username, password)
+	hub, err := registry.New(opts.RegistryBase, username, password)
 	if err != nil {
 		jww.ERROR.Fatalln(err)
 		return err
@@ -78,7 +91,6 @@ func DownloadImage(sourceImage, output, registryBase string) error {
 			jww.ERROR.Println(err)
 			return err
 		}
-		defer os.RemoveAll(TempDir)
 
 		out, err := os.Create(path.Join(where, "layer.tar"))
 		if err != nil {
@@ -91,13 +103,23 @@ func DownloadImage(sourceImage, output, registryBase string) error {
 		}
 	}
 
+	jww.INFO.Println("Download complete")
+
 	export, err := CreateExport(TempDir)
 	if err != nil {
 		fmt.Println(err)
 		return err
 	}
-	export.UnPackLayers(layers_sha, output)
-	jww.INFO.Printf("Done")
+
+	jww.INFO.Println("Unpacking...")
+
+	err = export.UnPackLayers(layers_sha, output)
+	if err != nil {
+		jww.INFO.Fatal(err)
+		return err
+	}
+
+	jww.INFO.Println("Done")
 	return nil
 }
 

api/export.go

@@ -32,6 +32,11 @@ func (e *Export) ExtractLayers() error {
 
 func (e *Export) UnPackLayers(order []string, layerDir string) error {
 
+	unpackmode := os.Getenv("UNPACK_MODE")
+	if unpackmode == "" {
+		unpackmode = "umoci"
+	}
+
 	err := os.MkdirAll(layerDir, 0755)
 	if err != nil {
 		return err
@@ -43,9 +48,15 @@ func (e *Export) UnPackLayers(order []string, layerDir string) error {
 			continue
 		}
 
-		out, err := extractTar(entry.LayerTarPath, layerDir)
+		err := ExtractLayer(&ExtractOpts{
+			Source:       entry.LayerTarPath,
+			Destination:  layerDir,
+			Compressed:   true,
+			KeepDirlinks: true,
+			Rootless:     false,
+			UnpackMode:   unpackmode})
 		if err != nil {
-			jww.INFO.Println(out)
+			jww.INFO.Println(err.Error())
 			return err
 		}
 

api/image.go

@@ -53,10 +53,18 @@ func (e *ExportedImage) ExtractLayerDir() error {
 	if err != nil {
 		return err
 	}
+	unpackmode := os.Getenv("UNPACK_MODE")
+	if unpackmode == "" {
+		unpackmode = "umoci"
+	}
 
-	out, err := extractTar(e.LayerTarPath, e.LayerDirPath)
-	if err != nil {
-		jww.INFO.Println(out)
+	if err := ExtractLayer(&ExtractOpts{
+		Source:       e.LayerTarPath,
+		Destination:  e.LayerDirPath,
+		Compressed:   true,
+		KeepDirlinks: true,
+		Rootless:     false,
+		UnpackMode:   unpackmode}); err != nil {
 		return err
 	}
 	return nil

api/util.go

@@ -1,21 +1,52 @@
 package api
 
 import (
+	"bufio"
+	"context"
+	"io"
 	"os"
-	"os/exec"
-
-	docker "github.com/fsouza/go-dockerclient"
 
 	"github.com/codegangsta/cli"
+	archive "github.com/containerd/containerd/archive"
+	dockerarchive "github.com/docker/docker/pkg/archive"
+	docker "github.com/fsouza/go-dockerclient"
+	layer "github.com/openSUSE/umoci/oci/layer"
 	jww "github.com/spf13/jwalterweatherman"
 )
 
-func extractTar(src, dest string) ([]byte, error) {
-	jww.INFO.Printf("Extracting: ", TarCmd, "--same-owner", "--xattrs", "--overwrite",
-		"--preserve-permissions", "-xf", src, "-C", dest)
-	cmd := exec.Command(TarCmd, "--same-owner", "--xattrs", "--overwrite",
-		"--preserve-permissions", "-xf", src, "-C", dest)
-	return cmd.CombinedOutput()
+type ExtractOpts struct {
+	Source, Destination                string
+	Compressed, KeepDirlinks, Rootless bool
+	UnpackMode                         string
+}
+
+func ExtractLayer(opts *ExtractOpts) error {
+	file, err := os.Open(opts.Source)
+	if err != nil {
+		return err
+	}
+	var r io.Reader
+	r = file
+
+	if opts.Compressed {
+		decompressedArchive, err := dockerarchive.DecompressStream(bufio.NewReader(file))
+		if err != nil {
+			return err
+		}
+		defer decompressedArchive.Close()
+		r = decompressedArchive
+	}
+
+	buf := bufio.NewReader(r)
+	switch opts.UnpackMode {
+	case "umoci": // more fixes are in there
+		return layer.UnpackLayer(opts.Destination, buf, &layer.MapOptions{KeepDirlinks: opts.KeepDirlinks, Rootless: opts.Rootless})
+	case "containerd": // more cross-compatible
+		_, err := archive.Apply(context.Background(), opts.Destination, buf)
+		return err
+	default: // moby way
+		return Untar(buf, opts.Destination, !opts.Compressed)
+	}
 }
 
 // PullImage pull the specified image

download.go

@@ -16,5 +16,5 @@ func downloadImage(c *cli.Context) error {
 		return cli.NewExitError("This command requires to argument: source-image output-folder(absolute)", 86)
 	}
 
-	return api.DownloadImage(sourceImage, output, "")
+	return api.DownloadImage(sourceImage, output, &api.DownloadOpts{KeepLayers: c.Bool("keep")})
 }

main.go

@@ -36,6 +36,12 @@ func main() {
 			Aliases: []string{"dl"},
 			Usage:   "Download and unpacks an image without using docker - Usage: download foo/barimage /foobar/folder",
 			Action:  downloadImage,
+			Flags: []cli.Flag{
+				cli.BoolFlag{
+					Name:  "keep",
+					Usage: "Keeps downloaded layers around (useful for debugging)",
+				},
+			},
 		},
 		{
 			Name:    "unpack",