Include hard-linked files as hard-links in the tarstream

This pulls in an updated Microsoft/go-winio in order to use
GetFileStandardInfo to recognise hard-linked files, added in
microsoft/go-winio#185

Consequently, this pulls in containerd v1.5.0-beta1, as v1.4.0 and
earlier have fork of go-winio's backuptar which has a different API in
go-winio now.

Signed-off-by: Paul "TBBle" Hampson <Paul.Hampson@Pobox.com>

go.mod

@@ -3,33 +3,20 @@ module github.com/Microsoft/hcsshim
 go 1.13
 
 require (
-	github.com/Microsoft/go-winio v0.4.16-0.20201130162521-d1ffc52c7331
-	github.com/containerd/cgroups v0.0.0-20200531161412-0dbf7f05ba59
-	github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1
-	github.com/containerd/containerd v1.3.2
-	github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc // indirect
-	github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448 // indirect
-	github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3
-	github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de
-	github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd
-	github.com/gogo/protobuf v1.3.1
-	github.com/golang/protobuf v1.3.2 // indirect
-	github.com/kr/pretty v0.1.0 // indirect
-	github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2 // indirect
-	github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f // indirect
-	github.com/opencontainers/runtime-spec v1.0.2
+	github.com/Microsoft/go-winio v0.4.17-0.20210211115548-6eac466e5fa3
+	github.com/containerd/cgroups v0.0.0-20200824123100-0b889c03f102
+	github.com/containerd/console v1.0.1
+	github.com/containerd/containerd v1.5.0-beta.1
+	github.com/containerd/go-runc v0.0.0-20200220073739-7016d3ce2328
+	github.com/containerd/ttrpc v1.0.2
+	github.com/containerd/typeurl v1.0.1
+	github.com/gogo/protobuf v1.3.2
+	github.com/opencontainers/runtime-spec v1.0.3-0.20200929063507-e6143ca7d51d
 	github.com/pkg/errors v0.9.1
-	github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7 // indirect
-	github.com/sirupsen/logrus v1.4.2
-	github.com/stretchr/testify v1.4.0 // indirect
+	github.com/sirupsen/logrus v1.7.0
 	github.com/urfave/cli v1.22.2
-	go.opencensus.io v0.22.0
-	golang.org/x/net v0.0.0-20191004110552-13f9640d40b9 // indirect
-	golang.org/x/sync v0.0.0-20190423024810-112230192c58
-	golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3
-	google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873 // indirect
-	google.golang.org/grpc v1.23.1
-	gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
-	gopkg.in/yaml.v2 v2.2.8 // indirect
-	gotest.tools v2.2.0+incompatible // indirect
+	go.opencensus.io v0.22.3
+	golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9
+	golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c
+	google.golang.org/grpc v1.30.0
 )

internal/wclayer/baselayerreader.go

@@ -186,6 +186,18 @@ func (r *baseLayerReader) Next() (path string, size int64, fileInfo *winio.FileB
 	return
 }
 
+func (r *baseLayerReader) LinkInfo() (uint32, *winio.FileIDInfo, error) {
+	fileStandardInfo, err := winio.GetFileStandardInfo(r.currentFile)
+	if err != nil {
+		return 0, nil, err
+	}
+	fileIDInfo, err := winio.GetFileID(r.currentFile)
+	if err != nil {
+		return 0, nil, err
+	}
+	return fileStandardInfo.NumberOfLinks, fileIDInfo, nil
+}
+
 func (r *baseLayerReader) Read(b []byte) (int, error) {
 	if r.backupReader == nil {
 		if r.currentFile == nil {

internal/wclayer/exportlayer.go

@@ -44,6 +44,8 @@ func ExportLayer(ctx context.Context, path string, exportFolderPath string, pare
 type LayerReader interface {
 	// Next advances to the next file and returns the name, size, and file info
 	Next() (string, int64, *winio.FileBasicInfo, error)
+	// LinkInfo returns the number of links and the file identifier for the current file.
+	LinkInfo() (uint32, *winio.FileIDInfo, error)
 	// Read reads data from the current file, in the format of a Win32 backup stream, and
 	// returns the number of bytes read.
 	Read(b []byte) (int, error)

internal/wclayer/legacy.go

@@ -294,6 +294,18 @@ func (r *legacyLayerReader) Next() (path string, size int64, fileInfo *winio.Fil
 	return
 }
 
+func (r *legacyLayerReader) LinkInfo() (uint32, *winio.FileIDInfo, error) {
+	fileStandardInfo, err := winio.GetFileStandardInfo(r.currentFile)
+	if err != nil {
+		return 0, nil, err
+	}
+	fileIDInfo, err := winio.GetFileID(r.currentFile)
+	if err != nil {
+		return 0, nil, err
+	}
+	return fileStandardInfo.NumberOfLinks, fileIDInfo, nil
+}
+
 func (r *legacyLayerReader) Read(b []byte) (int, error) {
 	if r.backupReader == nil {
 		if r.currentFile == nil {

pkg/ociwclayer/export.go

@@ -51,6 +51,8 @@ func ExportLayerToTar(ctx context.Context, w io.Writer, path string, parentLayer
 }
 
 func writeTarFromLayer(ctx context.Context, r hcsshim.LayerReader, w io.Writer) error {
+	linkRecords := make(map[[16]byte]string)
+
 	t := tar.NewWriter(w)
 	for {
 		select {
@@ -76,6 +78,27 @@ func writeTarFromLayer(ctx context.Context, r hcsshim.LayerReader, w io.Writer)
 				return err
 			}
 		} else {
+			numberOfLinks, fileIDInfo, err := r.LinkInfo()
+			if err != nil {
+				return err
+			}
+			if numberOfLinks > 1 {
+				if linkName, ok := linkRecords[fileIDInfo.FileID]; ok {
+					// We've seen this file before, by another name, so put a hardlink in the tar stream.
+					hdr := backuptar.BasicInfoHeader(name, 0, fileInfo)
+					hdr.Mode = 0644
+					hdr.Typeflag = tar.TypeLink
+					hdr.Linkname = linkName
+					if err := t.WriteHeader(hdr); err != nil {
+						return err
+					}
+					continue
+				}
+
+				// All subsequent names for this file will be hard-linked to this name
+				linkRecords[fileIDInfo.FileID] = filepath.ToSlash(name)
+			}
+
 			err = backuptar.WriteTarFileFromBackupStream(t, r, name, size, fileInfo)
 			if err != nil {
 				return err

test/go.mod

@@ -3,32 +3,20 @@ module github.com/Microsoft/hcsshim/test
 go 1.13
 
 require (
-	github.com/Microsoft/go-winio v0.4.16-0.20201130162521-d1ffc52c7331
-	github.com/Microsoft/hcsshim v0.8.7
-	github.com/blang/semver v3.1.0+incompatible // indirect
-	github.com/containerd/containerd v1.3.2
-	github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3
-	github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de
-	github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd
-	github.com/docker/distribution v0.0.0-20190905152932-14b96e55d84c // indirect
-	github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect
-	github.com/gogo/googleapis v1.2.0 // indirect
-	github.com/gogo/protobuf v1.3.1
-	github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce // indirect
-	github.com/hashicorp/go-multierror v0.0.0-20161216184304-ed905158d874 // indirect
-	github.com/imdario/mergo v0.3.8 // indirect
-	github.com/opencontainers/runtime-spec v1.0.2
+	github.com/Microsoft/go-winio v0.4.17-0.20210211115548-6eac466e5fa3
+	github.com/Microsoft/hcsshim v0.8.14
+	github.com/containerd/containerd v1.5.0-beta.1
+	github.com/containerd/go-runc v0.0.0-20200220073739-7016d3ce2328
+	github.com/containerd/ttrpc v1.0.2
+	github.com/containerd/typeurl v1.0.1
+	github.com/gogo/protobuf v1.3.2
+	github.com/opencontainers/runtime-spec v1.0.3-0.20200929063507-e6143ca7d51d
 	github.com/opencontainers/runtime-tools v0.0.0-20181011054405-1d69bd0f9c39
 	github.com/pkg/errors v0.9.1
-	github.com/sirupsen/logrus v1.4.2
-	github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8 // indirect
-	github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect
-	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
-	github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f // indirect
-	go.etcd.io/bbolt v1.3.3 // indirect
-	golang.org/x/sync v0.0.0-20190423024810-112230192c58
-	google.golang.org/grpc v1.23.1
-	k8s.io/cri-api v0.17.3
+	github.com/sirupsen/logrus v1.7.0
+	golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9
+	google.golang.org/grpc v1.30.0
+	k8s.io/cri-api v0.20.1
 )
 
 replace github.com/Microsoft/hcsshim => ../