From e286e8ca03ab831468b816fd532095ae7c3666cf Mon Sep 17 00:00:00 2001 From: "Paul \"TBBle\" Hampson" Date: Thu, 3 Dec 2020 00:32:02 +1100 Subject: [PATCH 1/3] Simple baseLayerReader to export parentless layers This is the inverse of the baseLayerWriter: It walks Files/ and UtilityVM/Files/ (if present) and ignores the rest of the layer data, as it will be recreated when the layer is imported. Signed-off-by: Paul "TBBle" Hampson --- cmd/wclayer/export.go | 2 +- internal/wclayer/baselayerreader.go | 206 ++++++++++++++++++ .../{baselayer.go => baselayerwriter.go} | 0 internal/wclayer/exportlayer.go | 5 + 4 files changed, 212 insertions(+), 1 deletion(-) create mode 100644 internal/wclayer/baselayerreader.go rename internal/wclayer/{baselayer.go => baselayerwriter.go} (100%) diff --git a/cmd/wclayer/export.go b/cmd/wclayer/export.go index d739e457de..f9cc08a309 100644 --- a/cmd/wclayer/export.go +++ b/cmd/wclayer/export.go @@ -40,7 +40,7 @@ var exportCommand = cli.Command{ return err } - layers, err := normalizeLayers(cliContext.StringSlice("layer"), true) + layers, err := normalizeLayers(cliContext.StringSlice("layer"), false) if err != nil { return err } diff --git a/internal/wclayer/baselayerreader.go b/internal/wclayer/baselayerreader.go new file mode 100644 index 0000000000..7a13c98949 --- /dev/null +++ b/internal/wclayer/baselayerreader.go @@ -0,0 +1,206 @@ +package wclayer + +import ( + "context" + "errors" + "io" + "os" + "path/filepath" + "strings" + "syscall" + + "github.com/Microsoft/go-winio" + "github.com/Microsoft/hcsshim/internal/longpath" + "github.com/Microsoft/hcsshim/internal/oc" + "go.opencensus.io/trace" +) + +type baseLayerReader struct { + ctx context.Context + s *trace.Span + root string + result chan *fileEntry + proceed chan bool + currentFile *os.File + backupReader *winio.BackupFileReader +} + +func newBaseLayerReader(ctx context.Context, root string, s *trace.Span) (r *baseLayerReader) { + r = &baseLayerReader{ + ctx: ctx, + s: s, + root: root, + result: make(chan *fileEntry), + proceed: make(chan bool), + } + go r.walk() + return r +} + +func (r *baseLayerReader) walkUntilCancelled() error { + root, err := longpath.LongAbs(r.root) + if err != nil { + return err + } + + r.root = root + + err = filepath.Walk(filepath.Join(r.root, filesPath), func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + // Indirect fix for https://github.com/moby/moby/issues/32838#issuecomment-343610048. + // Handle failure from what may be a golang bug in the conversion of + // UTF16 to UTF8 in files which are left in the recycle bin. Os.Lstat + // which is called by filepath.Walk will fail when a filename contains + // unicode characters. Skip the recycle bin regardless which is goodness. + if strings.EqualFold(path, filepath.Join(r.root, `Files\$Recycle.Bin`)) && info.IsDir() { + return filepath.SkipDir + } + + r.result <- &fileEntry{path, info, nil} + if !<-r.proceed { + return errorIterationCanceled + } + + return nil + }) + + if err == errorIterationCanceled { + return nil + } + + if err != nil { + return err + } + + utilityVMAbsPath := filepath.Join(r.root, utilityVMPath) + utilityVMFilesAbsPath := filepath.Join(r.root, utilityVMFilesPath) + + // Ignore a UtilityVM without Files, that's not _really_ a UtiltyVM + if _, err = os.Lstat(utilityVMFilesAbsPath); err != nil { + if os.IsNotExist(err) { + return io.EOF + } + return err + } + + err = filepath.Walk(utilityVMAbsPath, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + if path != utilityVMAbsPath && path != utilityVMFilesAbsPath && !hasPathPrefix(path, utilityVMFilesAbsPath) { + if info.IsDir() { + return filepath.SkipDir + } + return nil + } + + r.result <- &fileEntry{path, info, nil} + if !<-r.proceed { + return errorIterationCanceled + } + + return nil + }) + + if err == errorIterationCanceled { + return nil + } + + if err != nil { + return err + } + + return io.EOF +} + +func (r *baseLayerReader) walk() { + defer close(r.result) + if !<-r.proceed { + return + } + + err := r.walkUntilCancelled() + if err != nil { + for { + r.result <- &fileEntry{err: err} + if !<-r.proceed { + return + } + } + } +} + +func (r *baseLayerReader) reset() { + if r.backupReader != nil { + r.backupReader.Close() + r.backupReader = nil + } + if r.currentFile != nil { + r.currentFile.Close() + r.currentFile = nil + } +} + +func (r *baseLayerReader) Next() (path string, size int64, fileInfo *winio.FileBasicInfo, err error) { + r.reset() + r.proceed <- true + fe := <-r.result + if fe == nil { + err = errors.New("BaseLayerReader closed") + return + } + if fe.err != nil { + err = fe.err + return + } + + path, err = filepath.Rel(r.root, fe.path) + if err != nil { + return + } + + f, err := openFileOrDir(fe.path, syscall.GENERIC_READ, syscall.OPEN_EXISTING) + if err != nil { + return + } + defer func() { + if f != nil { + f.Close() + } + }() + + fileInfo, err = winio.GetFileBasicInfo(f) + if err != nil { + return + } + + size = fe.fi.Size() + r.backupReader = winio.NewBackupFileReader(f, true) + + r.currentFile = f + f = nil + return +} + +func (r *baseLayerReader) Read(b []byte) (int, error) { + if r.backupReader == nil { + if r.currentFile == nil { + return 0, io.EOF + } + return r.currentFile.Read(b) + } + return r.backupReader.Read(b) +} + +func (r *baseLayerReader) Close() (err error) { + defer r.s.End() + defer func() { oc.SetSpanStatus(r.s, err) }() + r.proceed <- false + <-r.result + r.reset() + return nil +} diff --git a/internal/wclayer/baselayer.go b/internal/wclayer/baselayerwriter.go similarity index 100% rename from internal/wclayer/baselayer.go rename to internal/wclayer/baselayerwriter.go diff --git a/internal/wclayer/exportlayer.go b/internal/wclayer/exportlayer.go index 08d6afd3b1..1f1d6849dd 100644 --- a/internal/wclayer/exportlayer.go +++ b/internal/wclayer/exportlayer.go @@ -68,6 +68,11 @@ func NewLayerReader(ctx context.Context, path string, parentLayerPaths []string) trace.StringAttribute("path", path), trace.StringAttribute("parentLayerPaths", strings.Join(parentLayerPaths, ", "))) + if len(parentLayerPaths) == 0 { + // This is a base layer. It gets exported differently. + return newBaseLayerReader(ctx, path, span), nil + } + exportPath, err := ioutil.TempDir("", "hcs") if err != nil { return nil, err From 9b47fa10908aebe5fb7e267c28be01714b6edc4b Mon Sep 17 00:00:00 2001 From: "Paul \"TBBle\" Hampson" Date: Sun, 6 Dec 2020 21:09:40 +1100 Subject: [PATCH 2/3] Introduce hcsshim.ConvertToBaseLayer This API allows turning any collection of files into a WCOW base layer. It will create the necessary files in Files/ for hcsshim.ProcessBaseLayer to function, validate the necessary files for hcsshim.ProcessUtilityVMImage if UtilityVM/ exists, and then call those two APIs to complete the process. Calling this on a directory containing an untarred base layer OCI tarball, gives a very similar outcome to passing the tar stream through ociwclayer.ImportLayer. The new API is used in `TestSCSIAddRemoveWCOW` to create nearly-empty base layers for the scratch layers attached and removed from the utility VM. A wclayer command is also introduced: `makebaselayer` for testing and validation purposes. Signed-off-by: Paul "TBBle" Hampson --- cmd/wclayer/makebaselayer.go | 24 ++++ cmd/wclayer/wclayer.go | 1 + internal/wclayer/converttobaselayer.go | 139 ++++++++++++++++++++ layer.go | 3 + test/functional/utilities/createuvm.go | 6 +- test/functional/utilities/scratch.go | 11 ++ test/functional/uvm_mem_backingtype_test.go | 2 +- test/functional/uvm_memory_test.go | 2 +- test/functional/uvm_properties_test.go | 2 +- test/functional/uvm_scsi_test.go | 4 +- test/functional/uvm_vsmb_test.go | 4 +- 11 files changed, 189 insertions(+), 9 deletions(-) create mode 100644 cmd/wclayer/makebaselayer.go create mode 100644 internal/wclayer/converttobaselayer.go diff --git a/cmd/wclayer/makebaselayer.go b/cmd/wclayer/makebaselayer.go new file mode 100644 index 0000000000..bdd3b62444 --- /dev/null +++ b/cmd/wclayer/makebaselayer.go @@ -0,0 +1,24 @@ +package main + +import ( + "path/filepath" + + "github.com/Microsoft/hcsshim" + "github.com/Microsoft/hcsshim/internal/appargs" + "github.com/urfave/cli" +) + +var makeBaseLayerCommand = cli.Command{ + Name: "makebaselayer", + Usage: "converts a directory containing 'Files/' into a base layer", + ArgsUsage: "", + Before: appargs.Validate(appargs.NonEmptyString), + Action: func(context *cli.Context) error { + path, err := filepath.Abs(context.Args().First()) + if err != nil { + return err + } + + return hcsshim.ConvertToBaseLayer(path) + }, +} diff --git a/cmd/wclayer/wclayer.go b/cmd/wclayer/wclayer.go index 1a4a9b2370..0e179f09ee 100644 --- a/cmd/wclayer/wclayer.go +++ b/cmd/wclayer/wclayer.go @@ -34,6 +34,7 @@ func main() { createCommand, exportCommand, importCommand, + makeBaseLayerCommand, mountCommand, removeCommand, unmountCommand, diff --git a/internal/wclayer/converttobaselayer.go b/internal/wclayer/converttobaselayer.go new file mode 100644 index 0000000000..65baf6d29e --- /dev/null +++ b/internal/wclayer/converttobaselayer.go @@ -0,0 +1,139 @@ +package wclayer + +import ( + "context" + "os" + "path/filepath" + "syscall" + + "github.com/Microsoft/hcsshim/internal/hcserror" + "github.com/Microsoft/hcsshim/internal/oc" + "github.com/Microsoft/hcsshim/internal/safefile" + "github.com/Microsoft/hcsshim/internal/winapi" + "github.com/pkg/errors" + "go.opencensus.io/trace" +) + +var hiveNames = []string{"DEFAULT", "SAM", "SECURITY", "SOFTWARE", "SYSTEM"} + +// Ensure the given file exists as an ordinary file, and create a zero-length file if not. +func ensureFile(path string, root *os.File) error { + stat, err := safefile.LstatRelative(path, root) + if err != nil && os.IsNotExist(err) { + newFile, err := safefile.OpenRelative(path, root, 0, syscall.FILE_SHARE_WRITE, winapi.FILE_CREATE, 0) + if err != nil { + return err + } + return newFile.Close() + } + + if err != nil { + return err + } + + if !stat.Mode().IsRegular() { + fullPath := filepath.Join(root.Name(), path) + return errors.Errorf("%s has unexpected file mode %s", fullPath, stat.Mode().String()) + } + + return nil +} + +func ensureBaseLayer(root *os.File) (hasUtilityVM bool, err error) { + // The base layer registry hives will be copied from here + const hiveSourcePath = "Files\\Windows\\System32\\config" + if err = safefile.MkdirAllRelative(hiveSourcePath, root); err != nil { + return + } + + for _, hiveName := range hiveNames { + hivePath := filepath.Join(hiveSourcePath, hiveName) + if err = ensureFile(hivePath, root); err != nil { + return + } + } + + stat, err := safefile.LstatRelative(utilityVMFilesPath, root) + + if os.IsNotExist(err) { + return false, nil + } + + if err != nil { + return + } + + if !stat.Mode().IsDir() { + fullPath := filepath.Join(root.Name(), utilityVMFilesPath) + return false, errors.Errorf("%s has unexpected file mode %s", fullPath, stat.Mode().String()) + } + + const bcdRelativePath = "EFI\\Microsoft\\Boot\\BCD" + + // Just check that this exists as a regular file. If it exists but is not a valid registry hive, + // ProcessUtilityVMImage will complain: + // "The registry could not read in, or write out, or flush, one of the files that contain the system's image of the registry." + bcdPath := filepath.Join(utilityVMFilesPath, bcdRelativePath) + + stat, err = safefile.LstatRelative(bcdPath, root) + if err != nil { + return false, errors.Wrapf(err, "UtilityVM must contain '%s'", bcdRelativePath) + } + + if !stat.Mode().IsRegular() { + fullPath := filepath.Join(root.Name(), bcdPath) + return false, errors.Errorf("%s has unexpected file mode %s", fullPath, stat.Mode().String()) + } + + return true, nil +} + +func convertToBaseLayer(ctx context.Context, root *os.File) error { + hasUtilityVM, err := ensureBaseLayer(root) + + if err != nil { + return err + } + + if err := ProcessBaseLayer(ctx, root.Name()); err != nil { + return err + } + + if !hasUtilityVM { + return nil + } + + err = safefile.EnsureNotReparsePointRelative(utilityVMPath, root) + if err != nil { + return err + } + + utilityVMPath := filepath.Join(root.Name(), utilityVMPath) + return ProcessUtilityVMImage(ctx, utilityVMPath) +} + +// ConvertToBaseLayer processes a candidate base layer, i.e. a directory +// containing the desired file content under Files/, and optionally the +// desired file content for a UtilityVM under UtilityVM/Files/ +func ConvertToBaseLayer(ctx context.Context, path string) (err error) { + title := "hcsshim::ConvertToBaseLayer" + ctx, span := trace.StartSpan(ctx, title) + defer span.End() + defer func() { oc.SetSpanStatus(span, err) }() + span.AddAttributes(trace.StringAttribute("path", path)) + + root, err := safefile.OpenRoot(path) + if err != nil { + return hcserror.New(err, title+" - failed", "") + } + defer func() { + if err2 := root.Close(); err == nil && err2 != nil { + err = hcserror.New(err2, title+" - failed", "") + } + }() + + if err = convertToBaseLayer(ctx, root); err != nil { + return hcserror.New(err, title+" - failed", "") + } + return nil +} diff --git a/layer.go b/layer.go index e323c8308d..afd1ddd0ae 100644 --- a/layer.go +++ b/layer.go @@ -70,6 +70,9 @@ func ProcessUtilityVMImage(path string) error { func UnprepareLayer(info DriverInfo, layerId string) error { return wclayer.UnprepareLayer(context.Background(), layerPath(&info, layerId)) } +func ConvertToBaseLayer(path string) error { + return wclayer.ConvertToBaseLayer(context.Background(), path) +} type DriverInfo struct { Flavour int diff --git a/test/functional/utilities/createuvm.go b/test/functional/utilities/createuvm.go index d88307bde9..662a509092 100644 --- a/test/functional/utilities/createuvm.go +++ b/test/functional/utilities/createuvm.go @@ -10,7 +10,7 @@ import ( // CreateWCOWUVM creates a WCOW utility VM with all default options. Returns the // UtilityVM object; folder used as its scratch -func CreateWCOWUVM(ctx context.Context, t *testing.T, id, image string) (*uvm.UtilityVM, []string, string) { +func CreateWCOWUVM(ctx context.Context, t *testing.T, id, image string) (*uvm.UtilityVM, string) { return CreateWCOWUVMFromOptsWithImage(ctx, t, uvm.NewDefaultOptionsWCOW(id, ""), image) } @@ -35,7 +35,7 @@ func CreateWCOWUVMFromOpts(ctx context.Context, t *testing.T, opts *uvm.OptionsW // CreateWCOWUVMFromOptsWithImage creates a WCOW utility VM with the passed opts // builds the LayerFolders based on `image`. Returns the UtilityVM object; // folder used as its scratch -func CreateWCOWUVMFromOptsWithImage(ctx context.Context, t *testing.T, opts *uvm.OptionsWCOW, image string) (*uvm.UtilityVM, []string, string) { +func CreateWCOWUVMFromOptsWithImage(ctx context.Context, t *testing.T, opts *uvm.OptionsWCOW, image string) (*uvm.UtilityVM, string) { if opts == nil { t.Fatal("opts must be set") } @@ -51,7 +51,7 @@ func CreateWCOWUVMFromOptsWithImage(ctx context.Context, t *testing.T, opts *uvm opts.LayerFolders = append(opts.LayerFolders, uvmLayers...) opts.LayerFolders = append(opts.LayerFolders, scratchDir) - return CreateWCOWUVMFromOpts(ctx, t, opts), uvmLayers, scratchDir + return CreateWCOWUVMFromOpts(ctx, t, opts), scratchDir } // CreateLCOWUVM with all default options. diff --git a/test/functional/utilities/scratch.go b/test/functional/utilities/scratch.go index e7cf11028a..8c115d6505 100644 --- a/test/functional/utilities/scratch.go +++ b/test/functional/utilities/scratch.go @@ -24,6 +24,17 @@ func init() { } } +// CreateWCOWBlankBaseLayer creates an as-blank-as-possible base WCOW layer, which +// can be used as the base of a WCOW RW layer when it's not going to be the container's +// scratch mount. +func CreateWCOWBlankBaseLayer(ctx context.Context, t *testing.T) []string { + tempDir := CreateTempDir(t) + if err := wclayer.ConvertToBaseLayer(context.Background(), tempDir); err != nil { + t.Fatalf("Failed ConvertToBaseLayer: %s", err) + } + return []string{tempDir} +} + // CreateWCOWBlankRWLayer uses HCS to create a temp test directory containing a // read-write layer containing a disk that can be used as a containers scratch // space. The VHD is created with VM group access diff --git a/test/functional/uvm_mem_backingtype_test.go b/test/functional/uvm_mem_backingtype_test.go index 03ff8a00ed..f43b57154e 100644 --- a/test/functional/uvm_mem_backingtype_test.go +++ b/test/functional/uvm_mem_backingtype_test.go @@ -21,7 +21,7 @@ func runMemStartLCOWTest(t *testing.T, opts *uvm.OptionsLCOW) { } func runMemStartWCOWTest(t *testing.T, opts *uvm.OptionsWCOW) { - u, _, scratchDir := testutilities.CreateWCOWUVMFromOptsWithImage(context.Background(), t, opts, "microsoft/nanoserver") + u, scratchDir := testutilities.CreateWCOWUVMFromOptsWithImage(context.Background(), t, opts, "microsoft/nanoserver") defer os.RemoveAll(scratchDir) u.Close() } diff --git a/test/functional/uvm_memory_test.go b/test/functional/uvm_memory_test.go index aad97c271b..f3571bd1a5 100644 --- a/test/functional/uvm_memory_test.go +++ b/test/functional/uvm_memory_test.go @@ -46,7 +46,7 @@ func TestUVMMemoryUpdateWCOW(t *testing.T) { opts := uvm.NewDefaultOptionsWCOW(t.Name(), "") opts.MemorySizeInMB = 1024 * 2 - u, _, uvmScratchDir := testutilities.CreateWCOWUVMFromOptsWithImage(ctx, t, opts, "mcr.microsoft.com/windows/nanoserver:1909") + u, uvmScratchDir := testutilities.CreateWCOWUVMFromOptsWithImage(ctx, t, opts, "mcr.microsoft.com/windows/nanoserver:1909") defer os.RemoveAll(uvmScratchDir) defer u.Close() diff --git a/test/functional/uvm_properties_test.go b/test/functional/uvm_properties_test.go index c9f732e4f1..9e814963df 100644 --- a/test/functional/uvm_properties_test.go +++ b/test/functional/uvm_properties_test.go @@ -28,7 +28,7 @@ func TestPropertiesGuestConnection_LCOW(t *testing.T) { func TestPropertiesGuestConnection_WCOW(t *testing.T) { testutilities.RequiresBuild(t, osversion.RS5) - uvm, _, uvmScratchDir := testutilities.CreateWCOWUVM(context.Background(), t, t.Name(), "microsoft/nanoserver") + uvm, uvmScratchDir := testutilities.CreateWCOWUVM(context.Background(), t, t.Name(), "microsoft/nanoserver") defer os.RemoveAll(uvmScratchDir) defer uvm.Close() diff --git a/test/functional/uvm_scsi_test.go b/test/functional/uvm_scsi_test.go index b25a0db38a..2d57401732 100644 --- a/test/functional/uvm_scsi_test.go +++ b/test/functional/uvm_scsi_test.go @@ -38,9 +38,11 @@ func TestSCSIAddRemoveLCOW(t *testing.T) { func TestSCSIAddRemoveWCOW(t *testing.T) { testutilities.RequiresBuild(t, osversion.RS5) // TODO make the image configurable to the build we're testing on - u, layers, uvmScratchDir := testutilities.CreateWCOWUVM(context.Background(), t, t.Name(), "mcr.microsoft.com/windows/nanoserver:1903") + u, uvmScratchDir := testutilities.CreateWCOWUVM(context.Background(), t, t.Name(), "mcr.microsoft.com/windows/nanoserver:1903") defer os.RemoveAll(uvmScratchDir) defer u.Close() + layers := testutilities.CreateWCOWBlankBaseLayer(context.Background(), t) + defer os.RemoveAll(layers[0]) testSCSIAddRemoveSingle(t, u, `c:\`, "windows", layers) } diff --git a/test/functional/uvm_vsmb_test.go b/test/functional/uvm_vsmb_test.go index 167aecccdf..0f105c38a5 100644 --- a/test/functional/uvm_vsmb_test.go +++ b/test/functional/uvm_vsmb_test.go @@ -18,7 +18,7 @@ import ( // TestVSMB tests adding/removing VSMB layers from a v2 Windows utility VM func TestVSMB(t *testing.T) { testutilities.RequiresBuild(t, osversion.RS5) - uvm, _, uvmScratchDir := testutilities.CreateWCOWUVM(context.Background(), t, t.Name(), "microsoft/nanoserver") + uvm, uvmScratchDir := testutilities.CreateWCOWUVM(context.Background(), t, t.Name(), "microsoft/nanoserver") defer os.RemoveAll(uvmScratchDir) defer uvm.Close() @@ -48,7 +48,7 @@ func TestVSMB_Writable(t *testing.T) { opts := uvm.NewDefaultOptionsWCOW(t.Name(), "") opts.NoWritableFileShares = true - vm, _, uvmScratchDir := testutilities.CreateWCOWUVMFromOptsWithImage(context.Background(), t, opts, "microsoft/nanoserver") + vm, uvmScratchDir := testutilities.CreateWCOWUVMFromOptsWithImage(context.Background(), t, opts, "microsoft/nanoserver") defer os.RemoveAll(uvmScratchDir) defer vm.Close() From d376404f92b38ed84f06a54d1d3191b3aae22545 Mon Sep 17 00:00:00 2001 From: "Paul \"TBBle\" Hampson" Date: Sat, 12 Dec 2020 23:33:11 +1100 Subject: [PATCH 3/3] Include hard-linked files as hard-links in the tarstream Signed-off-by: Paul "TBBle" Hampson --- internal/wclayer/baselayerreader.go | 12 ++++++++++++ internal/wclayer/exportlayer.go | 2 ++ internal/wclayer/legacy.go | 12 ++++++++++++ pkg/ociwclayer/export.go | 23 +++++++++++++++++++++++ 4 files changed, 49 insertions(+) diff --git a/internal/wclayer/baselayerreader.go b/internal/wclayer/baselayerreader.go index 7a13c98949..f5c8a57b35 100644 --- a/internal/wclayer/baselayerreader.go +++ b/internal/wclayer/baselayerreader.go @@ -186,6 +186,18 @@ func (r *baseLayerReader) Next() (path string, size int64, fileInfo *winio.FileB return } +func (r *baseLayerReader) LinkInfo() (uint32, *winio.FileIDInfo, error) { + fileStandardInfo, err := winio.GetFileStandardInfo(r.currentFile) + if err != nil { + return 0, nil, err + } + fileIDInfo, err := winio.GetFileID(r.currentFile) + if err != nil { + return 0, nil, err + } + return fileStandardInfo.NumberOfLinks, fileIDInfo, nil +} + func (r *baseLayerReader) Read(b []byte) (int, error) { if r.backupReader == nil { if r.currentFile == nil { diff --git a/internal/wclayer/exportlayer.go b/internal/wclayer/exportlayer.go index 1f1d6849dd..31e8af947d 100644 --- a/internal/wclayer/exportlayer.go +++ b/internal/wclayer/exportlayer.go @@ -46,6 +46,8 @@ func ExportLayer(ctx context.Context, path string, exportFolderPath string, pare type LayerReader interface { // Next advances to the next file and returns the name, size, and file info Next() (string, int64, *winio.FileBasicInfo, error) + // LinkInfo returns the number of links and the file identifier for the current file. + LinkInfo() (uint32, *winio.FileIDInfo, error) // Read reads data from the current file, in the format of a Win32 backup stream, and // returns the number of bytes read. Read(b []byte) (int, error) diff --git a/internal/wclayer/legacy.go b/internal/wclayer/legacy.go index 3e431877f8..e27b17d5c1 100644 --- a/internal/wclayer/legacy.go +++ b/internal/wclayer/legacy.go @@ -295,6 +295,18 @@ func (r *legacyLayerReader) Next() (path string, size int64, fileInfo *winio.Fil return } +func (r *legacyLayerReader) LinkInfo() (uint32, *winio.FileIDInfo, error) { + fileStandardInfo, err := winio.GetFileStandardInfo(r.currentFile) + if err != nil { + return 0, nil, err + } + fileIDInfo, err := winio.GetFileID(r.currentFile) + if err != nil { + return 0, nil, err + } + return fileStandardInfo.NumberOfLinks, fileIDInfo, nil +} + func (r *legacyLayerReader) Read(b []byte) (int, error) { if r.backupReader == nil { if r.currentFile == nil { diff --git a/pkg/ociwclayer/export.go b/pkg/ociwclayer/export.go index baa2dff3ee..1c2c82c701 100644 --- a/pkg/ociwclayer/export.go +++ b/pkg/ociwclayer/export.go @@ -51,6 +51,8 @@ func ExportLayerToTar(ctx context.Context, w io.Writer, path string, parentLayer } func writeTarFromLayer(ctx context.Context, r wclayer.LayerReader, w io.Writer) error { + linkRecords := make(map[[16]byte]string) + t := tar.NewWriter(w) for { select { @@ -76,6 +78,27 @@ func writeTarFromLayer(ctx context.Context, r wclayer.LayerReader, w io.Writer) return err } } else { + numberOfLinks, fileIDInfo, err := r.LinkInfo() + if err != nil { + return err + } + if numberOfLinks > 1 { + if linkName, ok := linkRecords[fileIDInfo.FileID]; ok { + // We've seen this file before, by another name, so put a hardlink in the tar stream. + hdr := backuptar.BasicInfoHeader(name, 0, fileInfo) + hdr.Mode = 0644 + hdr.Typeflag = tar.TypeLink + hdr.Linkname = linkName + if err := t.WriteHeader(hdr); err != nil { + return err + } + continue + } + + // All subsequent names for this file will be hard-linked to this name + linkRecords[fileIDInfo.FileID] = filepath.ToSlash(name) + } + err = backuptar.WriteTarFileFromBackupStream(t, r, name, size, fileInfo) if err != nil { return err