WCOW Base layer creation and export

cmd/wclayer/export.go

@@ -40,7 +40,7 @@ var exportCommand = cli.Command{
 			return err
 		}
 
-		layers, err := normalizeLayers(cliContext.StringSlice("layer"), true)
+		layers, err := normalizeLayers(cliContext.StringSlice("layer"), false)
 		if err != nil {
 			return err
 		}

cmd/wclayer/makebaselayer.go

@@ -0,0 +1,24 @@
+package main
+
+import (
+	"path/filepath"
+
+	"github.com/Microsoft/hcsshim"
+	"github.com/Microsoft/hcsshim/internal/appargs"
+	"github.com/urfave/cli"
+)
+
+var makeBaseLayerCommand = cli.Command{
+	Name:      "makebaselayer",
+	Usage:     "converts a directory containing 'Files/' into a base layer",
+	ArgsUsage: "<layer path>",
+	Before:    appargs.Validate(appargs.NonEmptyString),
+	Action: func(context *cli.Context) error {
+		path, err := filepath.Abs(context.Args().First())
+		if err != nil {
+			return err
+		}
+
+		return hcsshim.ConvertToBaseLayer(path)
+	},
+}

cmd/wclayer/wclayer.go

@@ -34,6 +34,7 @@ func main() {
 		createCommand,
 		exportCommand,
 		importCommand,
+		makeBaseLayerCommand,
 		mountCommand,
 		removeCommand,
 		unmountCommand,

internal/wclayer/baselayerreader.go

@@ -0,0 +1,218 @@
+package wclayer
+
+import (
+	"context"
+	"errors"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+	"syscall"
+
+	"github.com/Microsoft/go-winio"
+	"github.com/Microsoft/hcsshim/internal/longpath"
+	"github.com/Microsoft/hcsshim/internal/oc"
+	"go.opencensus.io/trace"
+)
+
+type baseLayerReader struct {
+	ctx          context.Context
+	s            *trace.Span
+	root         string
+	result       chan *fileEntry
+	proceed      chan bool
+	currentFile  *os.File
+	backupReader *winio.BackupFileReader
+}
+
+func newBaseLayerReader(ctx context.Context, root string, s *trace.Span) (r *baseLayerReader) {
+	r = &baseLayerReader{
+		ctx:     ctx,
+		s:       s,
+		root:    root,
+		result:  make(chan *fileEntry),
+		proceed: make(chan bool),
+	}
+	go r.walk()
+	return r
+}
+
+func (r *baseLayerReader) walkUntilCancelled() error {
+	root, err := longpath.LongAbs(r.root)
+	if err != nil {
+		return err
+	}
+
+	r.root = root
+
+	err = filepath.Walk(filepath.Join(r.root, filesPath), func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		// Indirect fix for https://github.com/moby/moby/issues/32838#issuecomment-343610048.
+		// Handle failure from what may be a golang bug in the conversion of
+		// UTF16 to UTF8 in files which are left in the recycle bin. Os.Lstat
+		// which is called by filepath.Walk will fail when a filename contains
+		// unicode characters. Skip the recycle bin regardless which is goodness.
+		if strings.EqualFold(path, filepath.Join(r.root, `Files\$Recycle.Bin`)) && info.IsDir() {
+			return filepath.SkipDir
+		}
+
+		r.result <- &fileEntry{path, info, nil}
+		if !<-r.proceed {
+			return errorIterationCanceled
+		}
+
+		return nil
+	})
+
+	if err == errorIterationCanceled {
+		return nil
+	}
+
+	if err != nil {
+		return err
+	}
+
+	utilityVMAbsPath := filepath.Join(r.root, utilityVMPath)
+	utilityVMFilesAbsPath := filepath.Join(r.root, utilityVMFilesPath)
+
+	// Ignore a UtilityVM without Files, that's not _really_ a UtiltyVM
+	if _, err = os.Lstat(utilityVMFilesAbsPath); err != nil {
+		if os.IsNotExist(err) {
+			return io.EOF
+		}
+		return err
+	}
+
+	err = filepath.Walk(utilityVMAbsPath, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if path != utilityVMAbsPath && path != utilityVMFilesAbsPath && !hasPathPrefix(path, utilityVMFilesAbsPath) {
+			if info.IsDir() {
+				return filepath.SkipDir
+			}
+			return nil
+		}
+
+		r.result <- &fileEntry{path, info, nil}
+		if !<-r.proceed {
+			return errorIterationCanceled
+		}
+
+		return nil
+	})
+
+	if err == errorIterationCanceled {
+		return nil
+	}
+
+	if err != nil {
+		return err
+	}
+
+	return io.EOF
+}
+
+func (r *baseLayerReader) walk() {
+	defer close(r.result)
+	if !<-r.proceed {
+		return
+	}
+
+	err := r.walkUntilCancelled()
+	if err != nil {
+		for {
+			r.result <- &fileEntry{err: err}
+			if !<-r.proceed {
+				return
+			}
+		}
+	}
+}
+
+func (r *baseLayerReader) reset() {
+	if r.backupReader != nil {
+		r.backupReader.Close()
+		r.backupReader = nil
+	}
+	if r.currentFile != nil {
+		r.currentFile.Close()
+		r.currentFile = nil
+	}
+}
+
+func (r *baseLayerReader) Next() (path string, size int64, fileInfo *winio.FileBasicInfo, err error) {
+	r.reset()
+	r.proceed <- true
+	fe := <-r.result
+	if fe == nil {
+		err = errors.New("BaseLayerReader closed")
+		return
+	}
+	if fe.err != nil {
+		err = fe.err
+		return
+	}
+
+	path, err = filepath.Rel(r.root, fe.path)
+	if err != nil {
+		return
+	}
+
+	f, err := openFileOrDir(fe.path, syscall.GENERIC_READ, syscall.OPEN_EXISTING)
+	if err != nil {
+		return
+	}
+	defer func() {
+		if f != nil {
+			f.Close()
+		}
+	}()
+
+	fileInfo, err = winio.GetFileBasicInfo(f)
+	if err != nil {
+		return
+	}
+
+	size = fe.fi.Size()
+	r.backupReader = winio.NewBackupFileReader(f, true)
+
+	r.currentFile = f
+	f = nil
+	return
+}
+
+func (r *baseLayerReader) LinkInfo() (uint32, *winio.FileIDInfo, error) {
+	fileStandardInfo, err := winio.GetFileStandardInfo(r.currentFile)
+	if err != nil {
+		return 0, nil, err
+	}
+	fileIDInfo, err := winio.GetFileID(r.currentFile)
+	if err != nil {
+		return 0, nil, err
+	}
+	return fileStandardInfo.NumberOfLinks, fileIDInfo, nil
+}
+
+func (r *baseLayerReader) Read(b []byte) (int, error) {
+	if r.backupReader == nil {
+		if r.currentFile == nil {
+			return 0, io.EOF
+		}
+		return r.currentFile.Read(b)
+	}
+	return r.backupReader.Read(b)
+}
+
+func (r *baseLayerReader) Close() (err error) {
+	defer r.s.End()
+	defer func() { oc.SetSpanStatus(r.s, err) }()
+	r.proceed <- false
+	<-r.result
+	r.reset()
+	return nil
+}