cimfs support: Add cimfs reader/writers

This PR is one of the multiple PRs that add support for using cimfs based layers for
containers. This PR adds the go wrappers over cimfs writer functions exported by cimfs.dll
and also includes a cimfs reader that can directly read data from cimfs files.

internal/cimfs/cim_test.go

@@ -0,0 +1,128 @@
+package cimfs
+
+import (
+ "bytes"
+ "fmt"
+ "io"
+ "io/ioutil"
+
+ "os"
+ "path/filepath"
+ "syscall"
+ "testing"
+ "time"
+
+ "github.com/Microsoft/go-winio"
+ "github.com/Microsoft/hcsshim/osversion"
+ "golang.org/x/sys/windows"
+)
+
+// A simple tuple type used to hold information about a file/directory that is created
+// during a test.
+type tuple struct {
+ filepath string
+ fileContents []byte
+ isDir bool
+}
+
+// A utility function to create a file/directory and write data to it in the given cim
+func createCimFileUtil(c *CimFsWriter, fileTuple tuple) error {
+ // create files inside the cim
+ fileInfo := &winio.FileBasicInfo{
+ CreationTime: syscall.NsecToFiletime(time.Now().UnixNano()),
+ LastAccessTime: syscall.NsecToFiletime(time.Now().UnixNano()),
+ LastWriteTime: syscall.NsecToFiletime(time.Now().UnixNano()),
+ ChangeTime: syscall.NsecToFiletime(time.Now().UnixNano()),
+ FileAttributes: 0,
+ }
+ if fileTuple.isDir {
+ fileInfo.FileAttributes = windows.FILE_ATTRIBUTE_DIRECTORY
+ }
+
+ if err := c.AddFile(filepath.FromSlash(fileTuple.filepath), fileInfo, int64(len(fileTuple.fileContents)), []byte{}, []byte{}, []byte{}); err != nil {
+ return err
+ }
+
+ if !fileTuple.isDir {
+ wc, err := c.Write(fileTuple.fileContents)
+ if err != nil || wc != len(fileTuple.fileContents) {
+ if err == nil {
+ return fmt.Errorf("unable to finish writing to file %s", fileTuple.filepath)
+ } else {
+ return err
+ }
+ }
+ }
+ return nil
+}
+
+// This test creates a cim, writes some files to it and then reads those files back.
+// The cim created by this test has only 3 files in the following tree
+// /
+// |- foobar.txt
+// |- foo
+// |--- bar.txt
+func TestCimReadWrite(t *testing.T) {
+
+ if osversion.Get().Version <= osversion.IRON_BUILD {
+ t.Skipf("cimfs tests should only be run on IRON+ builds")
+ }
+
+ testContents := []tuple{
+ {"foobar.txt", []byte("foobar test data"), false},
+ {"foo", []byte(""), true},
+ {"foo\\bar.txt", []byte("bar test data"), false},
+ }
+ cimName := "test.cim"
+ tempDir, err := ioutil.TempDir("", "cim-test")
+ if err != nil {
+ t.Fatalf("failed while creating temp directory: %s", err)
+ }
+ defer os.RemoveAll(tempDir)
+
+ c, err := Create(tempDir, "", cimName)
+ if err != nil {
+ t.Fatalf("failed while creating a cim: %s", err)
+ }
+
+ for _, ft := range testContents {
+ err := createCimFileUtil(c, ft)
+ if err != nil {
+ t.Fatalf("failed to create the file %s inside the cim:%s", ft.filepath, err)
+ }
+ }
+ c.Close()
+
+ // open and read the cim
+ cimReader, err := Open(filepath.Join(tempDir, cimName))
+ if err != nil {
+ t.Fatalf("failed while opening the cim: %s", err)
+ }
+
+ for _, ft := range testContents {
+ // make sure the size of byte array is larger than contents of the largest file
+ f, err := cimReader.Open(ft.filepath)
+ if err != nil {
+ t.Fatalf("unable to read file %s from the cim: %s", ft.filepath, err)
+ }
+ fileContents := make([]byte, f.Size())
+ if !ft.isDir {
+ // it is a file - read contents
+ rc, err := f.Read(fileContents)
+ if err != nil && err != io.EOF {
+ t.Fatalf("failure while reading file %s from cim: %s", ft.filepath, err)
+ } else if rc != len(ft.fileContents) {
+ t.Fatalf("couldn't read complete file contents for file: %s, read %d bytes, expected: %d", ft.filepath, rc, len(ft.fileContents))
+ } else if !bytes.Equal(fileContents[:rc], ft.fileContents) {
+ t.Fatalf("contents of file %s don't match", ft.filepath)
+ }
+ } else {
+ // it is a directory just do stat
+ _, err := f.Stat()
+ if err != nil {
+ t.Fatalf("failure while reading directory %s from cim: %s", ft.filepath, err)
+ }
+ }
+ }
+
+}

internal/cimfs/cim_writer_windows.go

@@ -0,0 +1,260 @@
+package cimfs
+
+import (
+ "io/ioutil"
+ "os"
+ "path/filepath"
+ "unsafe"
+
+ "github.com/Microsoft/go-winio"
+ "github.com/pkg/errors"
+ "golang.org/x/sys/windows"
+)
+
+// CimFsWriter represents a writer to a single CimFS filesystem instance. On disk, the
+// image is composed of a filesystem file and several object ID and region files.
+type CimFsWriter struct {
+ // name of this cim. Usually a <name>.cim file will be created to represent this cim.
+ name string
+ // handle is the CIMFS_IMAGE_HANDLE that must be passed when calling CIMFS APIs.
+ handle FsHandle
+ // name of the active file i.e the file to which we are currently writing.
+ activeName string
+ // stream to currently active file.
+ activeStream StreamHandle
+ // amount of bytes that can be written to the activeStream.
+ activeLeft int64
+}
+
+// creates a new cim image. The handle returned in the `cim.handle` variable can then be
+// used to do operations on this cim.
+func Create(imagePath string, oldFSName string, newFSName string) (_ *CimFsWriter, err error) {
+ var oldNameBytes *uint16
+ fsName := oldFSName
+ if oldFSName != "" {
+ oldNameBytes, err = windows.UTF16PtrFromString(oldFSName)
+ if err != nil {
+ return nil, err
+ }
+ }
+ var newNameBytes *uint16
+ if newFSName != "" {
+ fsName = newFSName
+ newNameBytes, err = windows.UTF16PtrFromString(newFSName)
+ if err != nil {
+ return nil, err
+ }
+ }
+ var handle FsHandle
+ if err := cimCreateImage(imagePath, oldNameBytes, newNameBytes, &handle); err != nil {
+ return nil, err
+ }
+ return &CimFsWriter{handle: handle, name: filepath.Join(imagePath, fsName)}, nil
+}
+
+// creates alternate stream of given size at the given path relative to the cim path. This
+// will replace the current active stream. Always, finish writing current active stream
+// and then create an alternate stream.
+func (c *CimFsWriter) CreateAlternateStream(path string, size uint64) (err error) {
+ err = c.closeStream()
+ if err != nil {
+ return err
+ }
+ err = cimCreateAlternateStream(c.handle, path, size, &c.activeStream)
+ if err != nil {
+ return err
+ }
+ return nil
+}
+
+// closes the currently active stream
+func (c *CimFsWriter) closeStream() error {
+ if c.activeStream == 0 {
+ return nil
+ }
+ err := cimCloseStream(c.activeStream)
+ if err == nil && c.activeLeft > 0 {
+ // Validate here because CimCloseStream does not and this improves error
+ // reporting. Otherwise the error will occur in the context of
+ // cimWriteStream.
+ err = errors.New("write truncated")
+ }
+ if err != nil {
+ err = &PathError{Cim: c.name, Op: "closeStream", Path: c.activeName, Err: err}
+ }
+ c.activeLeft = 0
+ c.activeStream = 0
+ c.activeName = ""
+ return err
+}
+
+// AddFile adds a new file to the image. The file is added at the specified path. After
+// calling this function, the file is set as the active stream for the image, so data can
+// be written by calling `Write`.
+func (c *CimFsWriter) AddFile(path string, info *winio.FileBasicInfo, fileSize int64, securityDescriptor []byte, extendedAttributes []byte, reparseData []byte) error {
+ err := c.closeStream()
+ if err != nil {
+ return err
+ }
+ fileMetadata := &cimFsFileMetadata{
+ Attributes: info.FileAttributes,
+ FileSize: fileSize,
+ CreationTime: info.CreationTime,
+ LastWriteTime: info.LastWriteTime,
+ ChangeTime: info.ChangeTime,
+ LastAccessTime: info.LastAccessTime,
+ }
+ if len(securityDescriptor) == 0 {
+ // Passing an empty security descriptor creates a CIM in a weird state.
+ // Pass the NULL DACL.
+ securityDescriptor = nullSd
+ }
+ fileMetadata.SecurityDescriptorBuffer = unsafe.Pointer(&securityDescriptor[0])
+ fileMetadata.SecurityDescriptorSize = uint32(len(securityDescriptor))
+ if len(reparseData) > 0 {
+ fileMetadata.ReparseDataBuffer = unsafe.Pointer(&reparseData[0])
+ fileMetadata.ReparseDataSize = uint32(len(reparseData))
+ }
+ if len(extendedAttributes) > 0 {
+ fileMetadata.ExtendedAttributes = unsafe.Pointer(&extendedAttributes[0])
+ fileMetadata.EACount = uint32(len(extendedAttributes))
+ }
+ err = cimCreateFile(c.handle, path, fileMetadata, &c.activeStream)
+ if err != nil {
+ return &PathError{Cim: c.name, Op: "addFile", Path: path, Err: err}
+ }
+ c.activeName = path
+ if info.FileAttributes&(windows.FILE_ATTRIBUTE_DIRECTORY) == 0 {
+ c.activeLeft = fileSize
+ }
+ return nil
+}
+
+// This is a helper function which reads the file on host at path `hostPath` and adds it
+// inside the cim at path `pathInCim`. If a file already exists inside cim at path
+// `pathInCim` it will be overwritten.
+func (c *CimFsWriter) AddFileFromPath(pathInCim, hostPath string, securityDescriptor []byte, extendedAttributes []byte, reparseData []byte) error {
+ f, err := os.Open(hostPath)
+ if err != nil {
+ return errors.Wrapf(err, "AddFileFromPath, can't open file: %s", hostPath)
+ }
+ defer f.Close()
+
+ basicInfo, err := winio.GetFileBasicInfo(f)
+ if err != nil {
+ return errors.Wrapf(err, "AddFileFromPath, failed to get file info for %s", hostPath)
+ }
+
+ replaceData, err := ioutil.ReadFile(hostPath)
+ if err != nil {
+ return errors.Wrapf(err, "AddFileFromPath, unable to read file %s", hostPath)
+ }
+ if err := c.AddFile(pathInCim, basicInfo, int64(len(replaceData)), securityDescriptor, extendedAttributes, reparseData); err != nil {
+ return err
+ }
+
+ if _, err := c.Write(replaceData); err != nil {
+ return &PathError{Cim: c.name, Op: "write", Path: c.activeName, Err: err}
+ }
+ return nil
+}
+
+// write writes bytes to the active stream.
+func (c *CimFsWriter) Write(p []byte) (int, error) {
+ if c.activeStream == 0 {
+ return 0, errors.New("no active stream")
+ }
+ if int64(len(p)) > c.activeLeft {
+ return 0, &PathError{Cim: c.name, Op: "write", Path: c.activeName, Err: errors.New("wrote too much")}
+ }
+ err := cimWriteStream(c.activeStream, uintptr(unsafe.Pointer(&p[0])), uint32(len(p)))
+ if err != nil {
+ err = &PathError{Cim: c.name, Op: "write", Path: c.activeName, Err: err}
+ return 0, err
+ }
+ c.activeLeft -= int64(len(p))
+ return len(p), nil
+}
+
+// Link adds a hard link from `oldPath` to `newPath` in the image.
+func (c *CimFsWriter) AddLink(oldPath string, newPath string) error {
+ err := c.closeStream()
+ if err != nil {
+ return err
+ }
+ err = cimCreateHardLink(c.handle, newPath, oldPath)
+ if err != nil {
+ err = &LinkError{Cim: c.name, Op: "addLink", Old: oldPath, New: newPath, Err: err}
+ }
+ return err
+}
+
+// Unlink deletes the file at `path` from the image.
+func (c *CimFsWriter) Unlink(path string) error {
+ err := c.closeStream()
+ if err != nil {
+ return err
+ }
+ err = cimDeletePath(c.handle, path)
+ if err != nil {
+ err = &PathError{Cim: c.name, Op: "unlink", Path: path, Err: err}
+ }
+ return err
+}
+
+func (c *CimFsWriter) commit() error {
+ err := c.closeStream()
+ if err != nil {
+ return err
+ }
+ err = cimCommitImage(c.handle)
+ if err != nil {
+ err = &OpError{Cim: c.name, Op: "commit", Err: err}
+ }
+ return err
+}
+
+// Close closes the CimFS filesystem.
+func (c *CimFsWriter) Close() error {
+ if c.handle == 0 {
+ return errors.New("invalid writer")
+ }
+ if err := c.commit(); err != nil {
+ return &OpError{Cim: c.name, Op: "commit", Err: err}
+ }
+ if err := cimCloseImage(c.handle); err != nil {
+ return &OpError{Cim: c.name, Op: "close", Err: err}
+ }
+ c.handle = 0
+ return nil
+}
+
+// DestroyCim finds out the region files, object files of this cim and then delete
+// the region files, object files and the <layer-id>.cim file itself.
+func DestroyCim(cimPath string) error {
+ regionFilePaths, err := GetRegionFilePaths(cimPath)
+ if err != nil {
+ return errors.Wrapf(err, "failed while destroying cim %s", cimPath)
+ }
+ objectFilePaths, err := GetObjectIdFilePaths(cimPath)
+ if err != nil {
+ return errors.Wrapf(err, "failed while destroying cim %s", cimPath)
+ }
+
+ for _, regFilePath := range regionFilePaths {
+ if err := os.Remove(regFilePath); err != nil {
+ return errors.Wrapf(err, "can't remove file: %s", regFilePath)
+ }
+ }
+
+ for _, objFilePath := range objectFilePaths {
+ if err := os.Remove(objFilePath); err != nil {
+ return errors.Wrapf(err, "can't remove file: %s", objFilePath)
+ }
+ }
+
+ if err := os.Remove(cimPath); err != nil {
+ return errors.Wrapf(err, "can't remove file: %s", cimPath)
+ }
+ return nil
+}