From af093968cb26e7f8c1ec0949c7218f32630d36cc Mon Sep 17 00:00:00 2001 From: Amit Barve Date: Tue, 19 Jan 2021 14:34:19 -0800 Subject: [PATCH] cimfs support: Add cimfs reader/writers This PR is one of the multiple PRs that add support for using cimfs based layers for containers. This PR adds the go wrappers over cimfs writer functions exported by cimfs.dll and also includes a cimfs reader that can directly read data from cimfs files. --- internal/cimfs/cim_reader_windows.go | 1070 ++++++++++++++++++++++++++ internal/cimfs/cim_test.go | 128 +++ internal/cimfs/cim_writer_windows.go | 260 +++++++ internal/cimfs/common.go | 109 +++ internal/cimfs/format/format.go | 260 +++++++ internal/cimfs/mount_cim.go | 83 ++ internal/cimfs/syscall_windows.go | 48 ++ internal/cimfs/zsyscall_windows.go | 238 ++++++ internal/schema2/cim_mount.go | 24 + internal/winapi/utils.go | 20 + osversion/windowsbuilds.go | 4 + 11 files changed, 2244 insertions(+) create mode 100644 internal/cimfs/cim_reader_windows.go create mode 100644 internal/cimfs/cim_test.go create mode 100644 internal/cimfs/cim_writer_windows.go create mode 100644 internal/cimfs/common.go create mode 100644 internal/cimfs/format/format.go create mode 100644 internal/cimfs/mount_cim.go create mode 100644 internal/cimfs/syscall_windows.go create mode 100644 internal/cimfs/zsyscall_windows.go create mode 100644 internal/schema2/cim_mount.go diff --git a/internal/cimfs/cim_reader_windows.go b/internal/cimfs/cim_reader_windows.go new file mode 100644 index 0000000000..c192956e68 --- /dev/null +++ b/internal/cimfs/cim_reader_windows.go @@ -0,0 +1,1070 @@ +package cimfs + +import ( + "bytes" + "encoding/binary" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "sync" + "unicode/utf16" + + winio "github.com/Microsoft/go-winio" + "github.com/Microsoft/hcsshim/internal/cimfs/format" + "github.com/Microsoft/hcsshim/internal/winapi" + "github.com/pkg/errors" + "golang.org/x/sys/windows" +) + +var ( + // ErrFileNotFound indicates that a file with the requested path was not + // found. + ErrFileNotFound = errors.New("no such file or directory") + // ErrNotADirectory indicates that a directory operation was attempted on a + // non-directory file. + ErrNotADirectory = errors.New("not a directory") + // ErrIsADirectory indicates that a non-directory operation was attempted on + // a directory. + ErrIsADirectory = errors.New("is a directory") +) + +type region struct { + f *os.File + size uint64 +} + +type fileTable []byte + +// A CimFsReader is a reader to a CIM file opened for read access. +type CimFsReader struct { + name string + reg []region + ftdes []format.FileTableDirectoryEntry + ftables []fileTable + upcase []uint16 + root *inode + cm sync.Mutex + inodeCache map[format.FileID]*inode + sdCache map[format.RegionOffset][]byte +} + +// A File is a file or directory within an open CIM. +type File struct { + r *CimFsReader + name string + sr streamReader + ino *inode +} + +type inode struct { + id format.FileID + file format.File + linkTable []byte + streamTable []byte +} + +type streamReader struct { + stream format.Stream + off uint64 + pe format.PeImage + pemappings []format.PeImageMapping + peinit bool +} + +// A Stream is an alternate data stream of a file within a CIM. +type Stream struct { + c *CimFsReader + r streamReader + fname string + name string +} + +func readBin(r io.Reader, v interface{}) error { + err := binary.Read(r, binary.LittleEndian, v) + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + return err +} + +func validateHeader(h *format.CommonHeader) error { + if !bytes.Equal(h.Magic[:], format.MagicValue[:]) { + return errors.New("not a cim file") + } + if h.Version.Major != format.CurrentVersion.Major { + return fmt.Errorf("unsupported cim version. cim version %v expected version %v", h.Version, format.CurrentVersion) + } + return nil +} + +func loadRegionSet(rs *format.RegionSet, imagePath string, reg []region) (int, error) { + for i := 0; i < int(rs.Count); i++ { + name := fmt.Sprintf("region_%v_%d", rs.ID, i) + rf, err := os.Open(filepath.Join(imagePath, name)) + if err != nil { + return 0, err + } + reg[i].f = rf + fi, err := rf.Stat() + if err != nil { + return 0, err + } + reg[i].size = uint64(fi.Size()) + var rh format.RegionHeader + err = readBin(rf, &rh) + if err != nil { + return 0, fmt.Errorf("reading region header %s: %s", name, err) + } + err = validateHeader(&rh.Common) + if err != nil { + return 0, fmt.Errorf("validating region header %s: %s", name, err) + } + } + return int(rs.Count), nil +} + +func readFilesystemHeader(f *os.File) (format.FilesystemHeader, error) { + var fsh format.FilesystemHeader + err := readBin(f, &fsh) + if err != nil { + return fsh, fmt.Errorf("reading filesystem header: %s", err) + } + err = validateHeader(&fsh.Common) + if err != nil { + return fsh, fmt.Errorf("validating filesystem header: %s", err) + } + return fsh, nil +} + +// Open opens a CIM file for read access. +func Open(p string) (_ *CimFsReader, err error) { + defer func() { + if err != nil { + err = &OpError{Cim: p, Op: "open", Err: err} + } + }() + f, err := os.Open(p) + if err != nil { + return nil, err + } + defer f.Close() + fsh, err := readFilesystemHeader(f) + if err != nil { + return nil, err + } + parents := make([]format.RegionSet, fsh.ParentCount) + err = readBin(f, parents) + if err != nil { + return nil, fmt.Errorf("reading parent region sets: %s", err) + } + regionCount := int(fsh.Regions.Count) + for i := range parents { + regionCount += int(parents[i].Count) + } + if regionCount == 0 || regionCount > 0x10000 { + return nil, fmt.Errorf("invalid region count %d", regionCount) + } + c := &CimFsReader{ + name: p, + reg: make([]region, regionCount), + upcase: make([]uint16, format.UpcaseTableLength), + inodeCache: make(map[format.FileID]*inode), + sdCache: make(map[format.RegionOffset][]byte), + } + defer func() { + if err != nil { + c.Close() + } + }() + + imagePath := filepath.Dir(p) + reg := c.reg + for i := range parents { + n, err := loadRegionSet(&parents[i], imagePath, reg) + if err != nil { + return nil, err + } + reg = reg[n:] + } + _, err = loadRegionSet(&fsh.Regions, imagePath, reg) + if err != nil { + return nil, err + } + var fs format.Filesystem + err = c.readBin(&fs, fsh.FilesystemOffset, 0) + if err != nil { + return nil, fmt.Errorf("reading filesystem info: %s", err) + } + c.ftables = make([]fileTable, fs.FileTableDirectoryLength) + c.ftdes = make([]format.FileTableDirectoryEntry, fs.FileTableDirectoryLength) + err = c.readBin(c.ftdes, fs.FileTableDirectoryOffset, 0) + if err != nil { + return nil, fmt.Errorf("reading file table directory: %s", err) + } + err = c.readBin(c.upcase, fs.UpcaseTableOffset, 0) + if err != nil { + return nil, fmt.Errorf("reading upcase table: %s", err) + } + c.root, err = c.getInode(fs.RootDirectory) + if err != nil { + return nil, fmt.Errorf("reading root directory: %s", err) + } + return c, nil +} + +// Close releases resources associated with the Cim. +func (cr *CimFsReader) Close() error { + for i := range cr.reg { + cr.reg[i].f.Close() + } + return nil +} + +func (cr *CimFsReader) objReader(o format.RegionOffset, off uint64, size int64) (*io.SectionReader, error) { + oi := int(o.RegionIndex()) + ob := o.ByteOffset() + if oi >= len(cr.reg) || ob == 0 { + return nil, fmt.Errorf("invalid region offset 0x%x", o) + } + reg := cr.reg[oi] + if ob > reg.size || off > reg.size-ob { + return nil, fmt.Errorf("%s: invalid region offset 0x%x", reg.f.Name(), o) + } + maxsize := int64(reg.size - ob - off) + if size < 0 { + size = maxsize + } else if size > maxsize { + return nil, fmt.Errorf("%s: invalid region size %x at offset 0x%x", reg.f.Name(), size, o) + } + return io.NewSectionReader(reg.f, int64(ob+off), int64(size)), nil +} + +func (cr *CimFsReader) readCounted(o format.RegionOffset, csize int) ([]byte, error) { + r, err := cr.objReader(o, 0, -1) + if err != nil { + return nil, err + } + var n uint32 + if csize == 2 { + var n16 uint16 + err = readBin(r, &n16) + if err != nil { + return nil, err + } + n = uint32(n16) + } else if csize == 4 { + var n32 uint32 + err = readBin(r, &n32) + if err != nil { + return nil, err + } + n = n32 + } else { + panic("invalid count size") + } + b := make([]byte, n) + _, err = io.ReadFull(r, b) + if err != nil { + return nil, err + } + return b, nil +} + +func (cr *CimFsReader) readOffsetFull(b []byte, o format.RegionOffset, off uint64) (int, error) { + r, err := cr.objReader(o, off, int64(len(b))) + if err != nil { + return 0, err + } + return io.ReadFull(r, b) +} + +func (cr *CimFsReader) readBin(v interface{}, o format.RegionOffset, off uint64) error { + r, err := cr.objReader(o, off, int64(binary.Size(v))) + if err != nil { + return err + } + return readBin(r, v) +} + +// OpenAt returns a file associated with path `p`, relative to `dirf`. If `dirf` +// is nil or `p` starts with '\\', then the path will be opened relative to the +// CIM root. +func (f *File) OpenAt(p string) (*File, error) { + return f.r.openAt(f, p) +} + +func (cr *CimFsReader) Root() *File { + return cr.newFile("\\", cr.root) +} + +func (cr *CimFsReader) Open(p string) (*File, error) { + return cr.Root().OpenAt(p) +} + +func (cr *CimFsReader) newFile(name string, ino *inode) *File { + return &File{ + r: cr, + name: name, + ino: ino, + sr: streamReader{stream: ino.file.DefaultStream}, + } +} + +func (f *File) WalkPath(p string) (*File, string, error) { + ino, walked, err := f.r.walkPath(f.ino, p) + if err != nil { + return nil, "", err + } + return f.r.newFile(p[:walked], ino), p[walked:], nil +} + +func (cr *CimFsReader) walkPath(ino *inode, p string) (*inode, int, error) { + walked := 0 + for walked < len(p) { + if !ino.IsDir() { + break + } + for n := 0; len(p) > walked+n && p[walked+n] == '\\'; n++ { + } + n := strings.IndexByte(p[walked:], '\\') + var ( + name string + next int + ) + if n < 0 { + name = p[walked:] + next = len(p) + } else { + name = p[walked : walked+n] + next = walked + n + 1 + } + if name != "" { + fid, err := cr.findChild(ino, name) + if err != nil { + return nil, 0, err + } + if fid == 0 { + break + } + ino, err = cr.getInode(fid) + if err != nil { + return nil, 0, err + } + } + walked = next + } + return ino, walked, nil +} + +func (cr *CimFsReader) openAt(dirf *File, p string) (_ *File, err error) { + fullp := p + var ino *inode + if len(p) > 0 && p[0] == '\\' { + ino = cr.root + } else { + ino = dirf.ino + fullp = dirf.name + "\\" + fullp + } + defer func() { + if err != nil { + err = &PathError{Cim: cr.name, Path: fullp, Op: "openat", Err: err} + } + }() + ino, walked, err := cr.walkPath(ino, p) + if err != nil { + return nil, err + } + if walked < len(p) { + if ino.IsDir() { + return nil, ErrFileNotFound + } else { + return nil, ErrNotADirectory + } + } + return cr.newFile(fullp, ino), nil +} + +func (cr *CimFsReader) readFile(id format.FileID, file *format.File) error { + if id == 0 { + return fmt.Errorf("invalid file ID %#x", id) + } + tid := uint64((id - 1) / format.FilesPerTable) + tfid := int((id - 1) % format.FilesPerTable) + if tid >= uint64(len(cr.ftdes)) || tfid >= int(cr.ftdes[tid].Count) { + return fmt.Errorf("invalid file ID %#x", id) + } + esize := int(cr.ftdes[tid].EntrySize) + if cr.ftables[tid] == nil { + b := make([]byte, esize*int(cr.ftdes[tid].Count)) + _, err := cr.readOffsetFull(b, cr.ftdes[tid].Offset, 0) + if err != nil { + return fmt.Errorf("reading file table %d: %s", tid, err) + } + cr.ftables[tid] = b + } + // This second copy is needed because the on-disk file size may be smaller + // than format.File). + b := make([]byte, binary.Size(file)) + copy(b, cr.ftables[tid][tfid*esize:(tfid+1)*esize]) + readBin(bytes.NewReader(b), file) + return nil +} + +func (cr *CimFsReader) getInode(id format.FileID) (*inode, error) { + cr.cm.Lock() + ino, ok := cr.inodeCache[id] + cr.cm.Unlock() + if ok { + return ino, nil + } + ino = &inode{ + id: id, + } + err := cr.readFile(id, &ino.file) + if err != nil { + return nil, err + } + switch typ := ino.file.DefaultStream.Type(); typ { + case format.StreamTypeData, + format.StreamTypeLinkTable, + format.StreamTypePeImage: + + default: + return nil, fmt.Errorf("unsupported stream type: %d", typ) + } + cr.cm.Lock() + cr.inodeCache[id] = ino + cr.cm.Unlock() + return ino, nil +} + +func (ino *inode) IsDir() bool { + return ino.file.DefaultStream.Type() == format.StreamTypeLinkTable +} + +// IsDir returns whether a file is a directory. +func (f *File) IsDir() bool { + return f.ino.IsDir() +} + +func (cr *CimFsReader) getSd(o format.RegionOffset) ([]byte, error) { + cr.cm.Lock() + sd, ok := cr.sdCache[o] + cr.cm.Unlock() + if ok { + return sd, nil + } + sd, err := cr.readCounted(o, 2) + if err != nil { + return nil, fmt.Errorf("reading security descriptor at 0x%x: %s", o, err) + } + cr.cm.Lock() + cr.sdCache[o] = sd + cr.cm.Unlock() + return sd, nil +} + +func (ino *inode) attributes() uint32 { + attr := uint32(0) + if ino.file.Flags&format.FileFlagReadOnly != 0 { + attr |= windows.FILE_ATTRIBUTE_READONLY + } + if ino.file.Flags&format.FileFlagHidden != 0 { + attr |= windows.FILE_ATTRIBUTE_HIDDEN + } + if ino.file.Flags&format.FileFlagSystem != 0 { + attr |= windows.FILE_ATTRIBUTE_SYSTEM + } + if ino.file.Flags&format.FileFlagArchive != 0 { + attr |= windows.FILE_ATTRIBUTE_ARCHIVE + } + if ino.file.Flags&format.FileFlagSparse != 0 { + attr |= windows.FILE_ATTRIBUTE_SPARSE_FILE + } + if ino.IsDir() { + attr |= windows.FILE_ATTRIBUTE_DIRECTORY + } + if ino.file.ReparseOffset != format.NullOffset { + attr |= windows.FILE_ATTRIBUTE_REPARSE_POINT + } + return attr +} + +func (cr *CimFsReader) stat(ino *inode) (*FileInfo, error) { + fi := &FileInfo{ + FileID: uint64(ino.id), + Attributes: ino.attributes(), + Size: int64(ino.file.DefaultStream.Size()), + CreationTime: Filetime(ino.file.CreationTime), + LastWriteTime: Filetime(ino.file.LastWriteTime), + ChangeTime: Filetime(ino.file.ChangeTime), + LastAccessTime: Filetime(ino.file.LastAccessTime), + } + + if ino.file.SdOffset != format.NullOffset { + sd, err := cr.getSd(ino.file.SdOffset) + if err != nil { + return nil, err + } + fi.SecurityDescriptor = sd + } + if ino.file.EaOffset != format.NullOffset { + b := make([]byte, ino.file.EaLength) + _, err := cr.readOffsetFull(b, ino.file.EaOffset, 0) + if err != nil { + return nil, fmt.Errorf("reading EA buffer at %#x: %s", ino.file.EaOffset, err) + } + fi.ExtendedAttributes = b + } + if ino.file.ReparseOffset != format.NullOffset { + b, err := cr.readCounted(ino.file.ReparseOffset, 2) + if err != nil { + return nil, fmt.Errorf("reading reparse buffer at %#x: %s", ino.file.EaOffset, err) + } + fi.ReparseData = b + } + return fi, nil +} + +func (f *File) Size() uint64 { + return f.ino.file.DefaultStream.Size() +} + +func (f *File) ReparseTag() uint32 { + return f.ino.file.ReparseTag +} + +// Stat returns a FileInfo for the file. +func (f *File) Stat() (*FileInfo, error) { + fi, err := f.r.stat(f.ino) + if err != nil { + err = &PathError{Cim: f.r.name, Path: f.name, Op: "stat", Err: err} + } + return fi, err +} + +func (cr *CimFsReader) getPESegment(sr *streamReader, off uint64) (int64, uint64, error) { + if !sr.peinit { + err := cr.readBin(&sr.pe, sr.stream.DataOffset, 0) + if err != nil { + return 0, 0, fmt.Errorf("reading PE image descriptor: %s", err) + } + sr.pe.DataLength &= 0x7fffffffffffffff // avoid returning negative lengths + sr.pemappings = make([]format.PeImageMapping, sr.pe.MappingCount) + err = cr.readBin(sr.pemappings, sr.stream.DataOffset, uint64(binary.Size(&sr.pe))) + if err != nil { + return 0, 0, fmt.Errorf("reading PE image mappings: %s", err) + } + sr.peinit = true + } + d := int64(0) + end := sr.pe.DataLength + for _, m := range sr.pemappings { + if uint64(m.FileOffset) > off { + end = uint64(m.FileOffset) + break + } + d = int64(m.Delta) + } + return d, end - off, nil +} + +func (cr *CimFsReader) readStream(sr *streamReader, b []byte) (_ int, err error) { + n := len(b) + rem := sr.stream.Size() - sr.off + if uint64(n) > rem { + n = int(rem) + } + if sr.stream.Sparse() { + for i := range b[:n] { + b[i] = 0 + } + return n, nil + } + ro := sr.stream.DataOffset + off := sr.off + if sr.stream.Type() == format.StreamTypePeImage { + delta, segrem, err := cr.getPESegment(sr, sr.off) + if err != nil { + return 0, err + } + if uint64(n) > segrem { + n = int(segrem) + } + ro = sr.pe.DataOffset + off += uint64(delta) + } + n, err = cr.readOffsetFull(b[:n], ro, off) + sr.off += uint64(n) + rem -= uint64(n) + if err == nil && rem == 0 { + err = io.EOF + } + return n, err +} + +func (f *File) Read(b []byte) (_ int, err error) { + defer func() { + if err != nil && err != io.EOF { + err = &PathError{Cim: f.r.name, Path: f.Name(), Op: "read", Err: err} + } + }() + if f.IsDir() { + return 0, ErrIsADirectory + } + return f.r.readStream(&f.sr, b) +} + +const ( + ltNameOffSize = 4 + ltNameLenSize = 2 + ltSizeOff = 0 + ltCountOff = 4 + ltEntryOff = 8 + fileIDSize = 4 + streamSize = 16 +) + +func parseName(b []byte, nos []byte, i int) ([]byte, error) { + size := uint32(len(b)) + no := binary.LittleEndian.Uint32(nos[i*ltNameOffSize:]) + if no > size-ltNameLenSize { + return nil, fmt.Errorf("invalid name offset %d > %d", no, size-ltNameLenSize) + } + nl := binary.LittleEndian.Uint16(b[no:]) + if mnl := (size - ltNameLenSize - no) / 2; uint32(nl) > mnl { + return nil, fmt.Errorf("invalid name length %d > %d", nl, mnl) + } + return b[no+ltNameLenSize : no+ltNameLenSize+uint32(nl)*2], nil +} + +// cmpcaseUtf8Utf16LE compares a UTF-8 string with a UTF-16LE encoded byte +// array, upcasing each rune through the upcase table. +func cmpcaseUtf8Utf16LE(a string, b []byte, upcase []uint16) int { + for _, ar := range a { + if len(b) == 0 { + return 1 + } + if int(ar) < len(upcase) { + ar = rune(upcase[int(ar)]) + } + br := rune(binary.LittleEndian.Uint16(b)) + bs := 2 + if utf16.IsSurrogate(br) { + if len(b) == bs { + return 1 // error? + } + br = utf16.DecodeRune(br, rune(binary.LittleEndian.Uint16(b[bs:]))) + if br == '\ufffd' { + return 1 // error? + } + bs += 2 + } else { + br = rune(upcase[int(br)]) + } + if ar < br { + return -1 + } else if ar > br { + return 1 + } + b = b[bs:] + } + if len(b) > 0 { + return -1 + } + return 0 +} + +func bsearchLinkTable(b []byte, esize int, name string, upcase []uint16) ([]byte, error) { + if len(b) == 0 { + return nil, nil + } + n := binary.LittleEndian.Uint32(b[ltCountOff:]) + es := b[ltEntryOff:] + nos := es[n*uint32(esize):] + lo := 0 + hi := int(n) + for hi > lo { + i := lo + (hi-lo)/2 + name16, err := parseName(b, nos, i) + if err != nil { + return nil, err + } + cmp := cmpcaseUtf8Utf16LE(name, name16, upcase) + if cmp < 0 { + hi = i + } else if cmp == 0 { + return es[i*esize : (i+1)*esize], nil + } else { + lo = i + 1 + } + } + return nil, nil +} + +func enumLinkTable(b []byte, esize int, f func(string, []byte) error) error { + if len(b) == 0 { + return nil + } + var lt format.LinkTable + r := bytes.NewReader(b) + readBin(r, <) + es := b[ltEntryOff:] + nos := es[lt.LinkCount*fileIDSize:] + for i := 0; i < int(lt.LinkCount); i++ { + name, err := parseName(b, nos, i) + if err != nil { + return err + } + if err := f(winapi.ParseUtf16LE(name), es[i*esize:(i+1)*esize]); err != nil { + return err + } + } + return nil +} + +func validateLinkTable(b []byte, esize int) error { + if len(b) < ltEntryOff { + return fmt.Errorf("invalid link table size %d", len(b)) + } + size := binary.LittleEndian.Uint32(b[ltSizeOff:]) + n := binary.LittleEndian.Uint32(b[ltCountOff:]) + if size < ltEntryOff { + return fmt.Errorf("invalid link table size %d", size) + } + if int64(size) > int64(len(b)) { + return fmt.Errorf("link table size mismatch %d < %d", len(b), size) + } + b = b[:size] + if maxn := size - ltEntryOff/(uint32(esize)+ltNameOffSize); maxn < n { + return fmt.Errorf("link table count mismatch %d < %d", maxn, n) + } + return nil +} + +func (cr *CimFsReader) getDirectoryTable(ino *inode) ([]byte, error) { + if !ino.IsDir() || ino.file.DefaultStream.Size() == 0 { + return nil, nil + } + cr.cm.Lock() + b := ino.linkTable + cr.cm.Unlock() + if b == nil { + b = make([]byte, ino.file.DefaultStream.Size()) + _, err := cr.readOffsetFull(b, ino.file.DefaultStream.DataOffset, 0) + if err != nil { + return nil, fmt.Errorf("reading directory link table: %s", err) + } + err = validateLinkTable(b, fileIDSize) + if err != nil { + return nil, err + } + cr.cm.Lock() + ino.linkTable = b + cr.cm.Unlock() + } + return b, nil +} + +func (cr *CimFsReader) findChild(ino *inode, name string) (format.FileID, error) { + table, err := cr.getDirectoryTable(ino) + if err != nil { + return 0, err + } + if table != nil { + b, err := bsearchLinkTable(table, fileIDSize, name, cr.upcase) + if err != nil { + return 0, err + } + if b != nil { + return format.FileID(binary.LittleEndian.Uint32(b)), nil + } + } + return 0, nil +} + +// Name returns the file's name. +func (f *File) Name() string { + return f.name +} + +// Readdir returns a slice of file names that are children of the directory. +// Fails if `f` is not a directory. +func (f *File) Readdir() (_ []string, err error) { + defer func() { + if err != nil { + err = &PathError{Cim: f.r.name, Path: f.name, Op: "readdir", Err: err} + } + }() + if !f.ino.IsDir() { + return nil, ErrNotADirectory + } + table, err := f.r.getDirectoryTable(f.ino) + if err != nil { + return nil, err + } + var names []string + err = enumLinkTable(table, fileIDSize, func(name string, fid []byte) error { + names = append(names, name) + return nil + }) + if err != nil { + return nil, err + } + return names, nil +} + +func (cr *CimFsReader) getStreamTable(ino *inode) ([]byte, error) { + if ino.file.StreamTableOffset == format.NullOffset { + return nil, nil + } + cr.cm.Lock() + table := ino.streamTable + cr.cm.Unlock() + if table == nil { + b, err := cr.readCounted(ino.file.StreamTableOffset, 4) + if err != nil { + return nil, fmt.Errorf("reading stream link table: %s", err) + } + err = validateLinkTable(b, streamSize) + if err != nil { + return nil, err + } + table = b + cr.cm.Lock() + ino.streamTable = table + cr.cm.Unlock() + } + return table, nil +} + +// Readstreams returns the names of the alternate data streams for a file. +func (f *File) Readstreams() (_ []string, err error) { + defer func() { + if err != nil { + err = &PathError{Cim: f.r.name, Path: f.name, Op: "readstreams", Err: err} + } + }() + table, err := f.r.getStreamTable(f.ino) + if err != nil { + return nil, err + } + var names []string + err = enumLinkTable(table, streamSize, func(name string, stream []byte) error { + names = append(names, name) + return nil + }) + if err != nil { + return nil, err + } + return names, nil +} + +// OpenStream opens an alternate data stream by name. +func (f *File) OpenStream(name string) (_ *Stream, err error) { + defer func() { + if err != nil { + err = &StreamError{Cim: f.r.name, Path: f.name, Stream: name, Op: "openstream", Err: err} + } + }() + table, err := f.r.getStreamTable(f.ino) + if err != nil { + return nil, err + } + if table != nil { + sb, err := bsearchLinkTable(table, streamSize, name, f.r.upcase) + if err != nil { + return nil, err + } + if sb != nil { + s := &Stream{c: f.r, fname: f.name, name: name} + readBin(bytes.NewReader(sb), &s.r.stream) + if typ := s.r.stream.Type(); typ != format.StreamTypeData { + return nil, fmt.Errorf("unsupported stream type %d", typ) + } + return s, nil + } + } + return nil, ErrFileNotFound +} + +type WalkFunc = func(file *File, stream *Stream) (bool, error) + +var SkipDir = errors.New("skip this directory") + +func Walk(f *File, fn WalkFunc) error { + enumStreams, err := fn(f, nil) + skipDir := false + if err == SkipDir { + skipDir = true + } else if err != nil { + return err + } + if enumStreams { + ss, err := f.Readstreams() + if err != nil { + return err + } + for _, sn := range ss { + s, err := f.OpenStream(sn) + if err != nil { + return err + } + _, err = fn(f, s) + if err != nil { + return err + } + } + } + if !f.IsDir() || skipDir { + return nil + } + names, err := f.Readdir() + if err != nil { + return err + } + for _, name := range names { + cf, err := f.OpenAt(name) + if err != nil { + return err + } + err = Walk(cf, fn) + if err != nil { + return err + } + } + return nil +} + +func (s *Stream) Read(b []byte) (int, error) { + n, err := s.c.readStream(&s.r, b) + if err != nil && err != io.EOF { + err = &StreamError{Cim: s.c.name, Path: s.fname, Stream: s.name, Op: "read", Err: err} + } + return n, err +} + +// Size returns the stream's length in bytes. +func (s *Stream) Size() uint64 { + return s.r.stream.Size() +} + +// Name returns the name of the stream. +func (s *Stream) Name() string { + return s.name +} + +// Returns the paths of all the region files associated with the cim at `cimPath`. +func GetRegionFilePaths(cimPath string) ([]string, error) { + f, err := os.Open(cimPath) + if err != nil { + return []string{}, errors.Wrapf(err, "GetRegionFilePaths: can't open file %s", cimPath) + } + defer f.Close() + + fsh, err := readFilesystemHeader(f) + if err != nil { + return []string{}, errors.Wrapf(err, "GetRegionFilePaths: failed while reading file %s", cimPath) + } + + paths := []string{} + for i := 0; i < int(fsh.Regions.Count); i++ { + path := filepath.Join(filepath.Dir(cimPath), fmt.Sprintf("region_%v_%d", fsh.Regions.ID, i)) + if _, err := os.Stat(path); err == nil { + paths = append(paths, path) + } + } + return paths, nil +} + +// Returns the paths of all the objectID files associated with the cim at `cimPath`. +func GetObjectIdFilePaths(cimPath string) ([]string, error) { + f, err := os.Open(cimPath) + if err != nil { + return []string{}, errors.Wrapf(err, "GetObjectIdFilePaths: can't open file %s", cimPath) + } + defer f.Close() + + fsh, err := readFilesystemHeader(f) + if err != nil { + return []string{}, errors.Wrapf(err, "GetObjectIdFilePaths: failed while reading file %s", cimPath) + } + + paths := []string{} + for i := 0; i < int(fsh.Regions.Count); i++ { + path := filepath.Join(filepath.Dir(cimPath), fmt.Sprintf("objectid_%v_%d", fsh.Regions.ID, i)) + if _, err := os.Stat(path); err == nil { + paths = append(paths, path) + } + + } + return paths, nil +} + +// fetchFileFromCim is an utility function that reads the file at path `filePath` inside +// the cim `cimPath` and copies it at `destinationPath`. +func FetchFileFromCim(cimPath, filePath, destinationPath string) (err error) { + // open the cim file and read it. + cimReader, err := Open(cimPath) + if err != nil { + return errors.Wrapf(err, "failed to open the cim %s", cimPath) + } + defer func() { + if err2 := cimReader.Close(); err == nil { + err = err2 + } + }() + + cimFile, err := cimReader.Open(filePath) + if err != nil { + return errors.Wrapf(err, "FetchFileFromCim, can't open file %s from the cim", filePath) + } + fileData := make([]byte, cimFile.Size()) + rc, err := cimFile.Read(fileData) + if err != nil && err != io.EOF { + return errors.Wrapf(err, "FetchFileFromCim, read cim failed") + } else if uint64(rc) != cimFile.Size() { + return errors.Wrapf(err, "FetchFileFromCim, read truncated for file %s", filePath) + } + + // create the destination file and write to it. + destinationFile, err := os.Create(destinationPath) + if err != nil { + return errors.Wrapf(err, "FetchFileFromCim, can't create file %s", destinationPath) + } + defer func() { + if err2 := destinationFile.Close(); err == nil { + err = err2 + } + }() + wc, err := destinationFile.Write(fileData) + if err != nil { + return errors.Wrapf(err, "FetchFileFromCim, can't write to file %s", destinationPath) + } else if wc != rc { + return errors.Wrapf(err, "FetchFileFromCim, write truncated for file %s", destinationPath) + } + + // set the basic file information + fstats, err := cimFile.Stat() + if err != nil { + return errors.Wrapf(err, "FetchFileFromCim, stat call failed for file %s", filePath) + } + + basicInfo := &winio.FileBasicInfo{ + CreationTime: fstats.CreationTime.toSyscallFiletime(), + LastAccessTime: fstats.LastAccessTime.toSyscallFiletime(), + LastWriteTime: fstats.LastWriteTime.toSyscallFiletime(), + ChangeTime: fstats.ChangeTime.toSyscallFiletime(), + FileAttributes: fstats.Attributes, + } + if err := winio.SetFileBasicInfo(destinationFile, basicInfo); err != nil { + return errors.Wrapf(err, "FetchFileFromCim, failed to set basic file info for %s", destinationPath) + } + // TODO(ambarve):Ideally we should handle SecurityDescriptor, reparseData and + // extendedAttributes here but legacy layer files always pass empty values for + // these so this is okay for now. + + return +} diff --git a/internal/cimfs/cim_test.go b/internal/cimfs/cim_test.go new file mode 100644 index 0000000000..65989a58ce --- /dev/null +++ b/internal/cimfs/cim_test.go @@ -0,0 +1,128 @@ +package cimfs + +import ( + "bytes" + "fmt" + "io" + "io/ioutil" + + "os" + "path/filepath" + "syscall" + "testing" + "time" + + "github.com/Microsoft/go-winio" + "github.com/Microsoft/hcsshim/osversion" + "golang.org/x/sys/windows" +) + +// A simple tuple type used to hold information about a file/directory that is created +// during a test. +type tuple struct { + filepath string + fileContents []byte + isDir bool +} + +// A utility function to create a file/directory and write data to it in the given cim +func createCimFileUtil(c *CimFsWriter, fileTuple tuple) error { + // create files inside the cim + fileInfo := &winio.FileBasicInfo{ + CreationTime: syscall.NsecToFiletime(time.Now().UnixNano()), + LastAccessTime: syscall.NsecToFiletime(time.Now().UnixNano()), + LastWriteTime: syscall.NsecToFiletime(time.Now().UnixNano()), + ChangeTime: syscall.NsecToFiletime(time.Now().UnixNano()), + FileAttributes: 0, + } + if fileTuple.isDir { + fileInfo.FileAttributes = windows.FILE_ATTRIBUTE_DIRECTORY + } + + if err := c.AddFile(filepath.FromSlash(fileTuple.filepath), fileInfo, int64(len(fileTuple.fileContents)), []byte{}, []byte{}, []byte{}); err != nil { + return err + } + + if !fileTuple.isDir { + wc, err := c.Write(fileTuple.fileContents) + if err != nil || wc != len(fileTuple.fileContents) { + if err == nil { + return fmt.Errorf("unable to finish writing to file %s", fileTuple.filepath) + } else { + return err + } + } + } + return nil +} + +// This test creates a cim, writes some files to it and then reads those files back. +// The cim created by this test has only 3 files in the following tree +// / +// |- foobar.txt +// |- foo +// |--- bar.txt +func TestCimReadWrite(t *testing.T) { + + if osversion.Get().Version <= osversion.IRON_BUILD { + t.Skipf("cimfs tests should only be run on IRON+ builds") + } + + testContents := []tuple{ + {"foobar.txt", []byte("foobar test data"), false}, + {"foo", []byte(""), true}, + {"foo\\bar.txt", []byte("bar test data"), false}, + } + cimName := "test.cim" + tempDir, err := ioutil.TempDir("", "cim-test") + if err != nil { + t.Fatalf("failed while creating temp directory: %s", err) + } + defer os.RemoveAll(tempDir) + + c, err := Create(tempDir, "", cimName) + if err != nil { + t.Fatalf("failed while creating a cim: %s", err) + } + + for _, ft := range testContents { + err := createCimFileUtil(c, ft) + if err != nil { + t.Fatalf("failed to create the file %s inside the cim:%s", ft.filepath, err) + } + } + c.Close() + + // open and read the cim + cimReader, err := Open(filepath.Join(tempDir, cimName)) + if err != nil { + t.Fatalf("failed while opening the cim: %s", err) + } + + for _, ft := range testContents { + // make sure the size of byte array is larger than contents of the largest file + f, err := cimReader.Open(ft.filepath) + if err != nil { + t.Fatalf("unable to read file %s from the cim: %s", ft.filepath, err) + } + fileContents := make([]byte, f.Size()) + if !ft.isDir { + // it is a file - read contents + rc, err := f.Read(fileContents) + if err != nil && err != io.EOF { + t.Fatalf("failure while reading file %s from cim: %s", ft.filepath, err) + } else if rc != len(ft.fileContents) { + t.Fatalf("couldn't read complete file contents for file: %s, read %d bytes, expected: %d", ft.filepath, rc, len(ft.fileContents)) + } else if !bytes.Equal(fileContents[:rc], ft.fileContents) { + t.Fatalf("contents of file %s don't match", ft.filepath) + } + } else { + // it is a directory just do stat + _, err := f.Stat() + if err != nil { + t.Fatalf("failure while reading directory %s from cim: %s", ft.filepath, err) + } + } + } + +} diff --git a/internal/cimfs/cim_writer_windows.go b/internal/cimfs/cim_writer_windows.go new file mode 100644 index 0000000000..9b5bda929f --- /dev/null +++ b/internal/cimfs/cim_writer_windows.go @@ -0,0 +1,260 @@ +package cimfs + +import ( + "io/ioutil" + "os" + "path/filepath" + "unsafe" + + "github.com/Microsoft/go-winio" + "github.com/pkg/errors" + "golang.org/x/sys/windows" +) + +// CimFsWriter represents a writer to a single CimFS filesystem instance. On disk, the +// image is composed of a filesystem file and several object ID and region files. +type CimFsWriter struct { + // name of this cim. Usually a .cim file will be created to represent this cim. + name string + // handle is the CIMFS_IMAGE_HANDLE that must be passed when calling CIMFS APIs. + handle FsHandle + // name of the active file i.e the file to which we are currently writing. + activeName string + // stream to currently active file. + activeStream StreamHandle + // amount of bytes that can be written to the activeStream. + activeLeft int64 +} + +// creates a new cim image. The handle returned in the `cim.handle` variable can then be +// used to do operations on this cim. +func Create(imagePath string, oldFSName string, newFSName string) (_ *CimFsWriter, err error) { + var oldNameBytes *uint16 + fsName := oldFSName + if oldFSName != "" { + oldNameBytes, err = windows.UTF16PtrFromString(oldFSName) + if err != nil { + return nil, err + } + } + var newNameBytes *uint16 + if newFSName != "" { + fsName = newFSName + newNameBytes, err = windows.UTF16PtrFromString(newFSName) + if err != nil { + return nil, err + } + } + var handle FsHandle + if err := cimCreateImage(imagePath, oldNameBytes, newNameBytes, &handle); err != nil { + return nil, err + } + return &CimFsWriter{handle: handle, name: filepath.Join(imagePath, fsName)}, nil +} + +// creates alternate stream of given size at the given path relative to the cim path. This +// will replace the current active stream. Always, finish writing current active stream +// and then create an alternate stream. +func (c *CimFsWriter) CreateAlternateStream(path string, size uint64) (err error) { + err = c.closeStream() + if err != nil { + return err + } + err = cimCreateAlternateStream(c.handle, path, size, &c.activeStream) + if err != nil { + return err + } + return nil +} + +// closes the currently active stream +func (c *CimFsWriter) closeStream() error { + if c.activeStream == 0 { + return nil + } + err := cimCloseStream(c.activeStream) + if err == nil && c.activeLeft > 0 { + // Validate here because CimCloseStream does not and this improves error + // reporting. Otherwise the error will occur in the context of + // cimWriteStream. + err = errors.New("write truncated") + } + if err != nil { + err = &PathError{Cim: c.name, Op: "closeStream", Path: c.activeName, Err: err} + } + c.activeLeft = 0 + c.activeStream = 0 + c.activeName = "" + return err +} + +// AddFile adds a new file to the image. The file is added at the specified path. After +// calling this function, the file is set as the active stream for the image, so data can +// be written by calling `Write`. +func (c *CimFsWriter) AddFile(path string, info *winio.FileBasicInfo, fileSize int64, securityDescriptor []byte, extendedAttributes []byte, reparseData []byte) error { + err := c.closeStream() + if err != nil { + return err + } + fileMetadata := &cimFsFileMetadata{ + Attributes: info.FileAttributes, + FileSize: fileSize, + CreationTime: info.CreationTime, + LastWriteTime: info.LastWriteTime, + ChangeTime: info.ChangeTime, + LastAccessTime: info.LastAccessTime, + } + if len(securityDescriptor) == 0 { + // Passing an empty security descriptor creates a CIM in a weird state. + // Pass the NULL DACL. + securityDescriptor = nullSd + } + fileMetadata.SecurityDescriptorBuffer = unsafe.Pointer(&securityDescriptor[0]) + fileMetadata.SecurityDescriptorSize = uint32(len(securityDescriptor)) + if len(reparseData) > 0 { + fileMetadata.ReparseDataBuffer = unsafe.Pointer(&reparseData[0]) + fileMetadata.ReparseDataSize = uint32(len(reparseData)) + } + if len(extendedAttributes) > 0 { + fileMetadata.ExtendedAttributes = unsafe.Pointer(&extendedAttributes[0]) + fileMetadata.EACount = uint32(len(extendedAttributes)) + } + err = cimCreateFile(c.handle, path, fileMetadata, &c.activeStream) + if err != nil { + return &PathError{Cim: c.name, Op: "addFile", Path: path, Err: err} + } + c.activeName = path + if info.FileAttributes&(windows.FILE_ATTRIBUTE_DIRECTORY) == 0 { + c.activeLeft = fileSize + } + return nil +} + +// This is a helper function which reads the file on host at path `hostPath` and adds it +// inside the cim at path `pathInCim`. If a file already exists inside cim at path +// `pathInCim` it will be overwritten. +func (c *CimFsWriter) AddFileFromPath(pathInCim, hostPath string, securityDescriptor []byte, extendedAttributes []byte, reparseData []byte) error { + f, err := os.Open(hostPath) + if err != nil { + return errors.Wrapf(err, "AddFileFromPath, can't open file: %s", hostPath) + } + defer f.Close() + + basicInfo, err := winio.GetFileBasicInfo(f) + if err != nil { + return errors.Wrapf(err, "AddFileFromPath, failed to get file info for %s", hostPath) + } + + replaceData, err := ioutil.ReadFile(hostPath) + if err != nil { + return errors.Wrapf(err, "AddFileFromPath, unable to read file %s", hostPath) + } + if err := c.AddFile(pathInCim, basicInfo, int64(len(replaceData)), securityDescriptor, extendedAttributes, reparseData); err != nil { + return err + } + + if _, err := c.Write(replaceData); err != nil { + return &PathError{Cim: c.name, Op: "write", Path: c.activeName, Err: err} + } + return nil +} + +// write writes bytes to the active stream. +func (c *CimFsWriter) Write(p []byte) (int, error) { + if c.activeStream == 0 { + return 0, errors.New("no active stream") + } + if int64(len(p)) > c.activeLeft { + return 0, &PathError{Cim: c.name, Op: "write", Path: c.activeName, Err: errors.New("wrote too much")} + } + err := cimWriteStream(c.activeStream, uintptr(unsafe.Pointer(&p[0])), uint32(len(p))) + if err != nil { + err = &PathError{Cim: c.name, Op: "write", Path: c.activeName, Err: err} + return 0, err + } + c.activeLeft -= int64(len(p)) + return len(p), nil +} + +// Link adds a hard link from `oldPath` to `newPath` in the image. +func (c *CimFsWriter) AddLink(oldPath string, newPath string) error { + err := c.closeStream() + if err != nil { + return err + } + err = cimCreateHardLink(c.handle, newPath, oldPath) + if err != nil { + err = &LinkError{Cim: c.name, Op: "addLink", Old: oldPath, New: newPath, Err: err} + } + return err +} + +// Unlink deletes the file at `path` from the image. +func (c *CimFsWriter) Unlink(path string) error { + err := c.closeStream() + if err != nil { + return err + } + err = cimDeletePath(c.handle, path) + if err != nil { + err = &PathError{Cim: c.name, Op: "unlink", Path: path, Err: err} + } + return err +} + +func (c *CimFsWriter) commit() error { + err := c.closeStream() + if err != nil { + return err + } + err = cimCommitImage(c.handle) + if err != nil { + err = &OpError{Cim: c.name, Op: "commit", Err: err} + } + return err +} + +// Close closes the CimFS filesystem. +func (c *CimFsWriter) Close() error { + if c.handle == 0 { + return errors.New("invalid writer") + } + if err := c.commit(); err != nil { + return &OpError{Cim: c.name, Op: "commit", Err: err} + } + if err := cimCloseImage(c.handle); err != nil { + return &OpError{Cim: c.name, Op: "close", Err: err} + } + c.handle = 0 + return nil +} + +// DestroyCim finds out the region files, object files of this cim and then delete +// the region files, object files and the .cim file itself. +func DestroyCim(cimPath string) error { + regionFilePaths, err := GetRegionFilePaths(cimPath) + if err != nil { + return errors.Wrapf(err, "failed while destroying cim %s", cimPath) + } + objectFilePaths, err := GetObjectIdFilePaths(cimPath) + if err != nil { + return errors.Wrapf(err, "failed while destroying cim %s", cimPath) + } + + for _, regFilePath := range regionFilePaths { + if err := os.Remove(regFilePath); err != nil { + return errors.Wrapf(err, "can't remove file: %s", regFilePath) + } + } + + for _, objFilePath := range objectFilePaths { + if err := os.Remove(objFilePath); err != nil { + return errors.Wrapf(err, "can't remove file: %s", objFilePath) + } + } + + if err := os.Remove(cimPath); err != nil { + return errors.Wrapf(err, "can't remove file: %s", cimPath) + } + return nil +} diff --git a/internal/cimfs/common.go b/internal/cimfs/common.go new file mode 100644 index 0000000000..2e88ad8f71 --- /dev/null +++ b/internal/cimfs/common.go @@ -0,0 +1,109 @@ +package cimfs + +import ( + "syscall" + "time" +) + +var ( + // Equivalent to SDDL of "D:NO_ACCESS_CONTROL" + nullSd = []byte{1, 0, 4, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} +) + +// 100ns units between Windows NT epoch (Jan 1 1601) and Unix epoch (Jan 1 1970) +const epochDelta = 116444736000000000 + +// Filetime is a Windows FILETIME, in 100-ns units since January 1, 1601. +type Filetime int64 + +// Time returns a Go time equivalent to `ft`. +func (ft Filetime) Time() time.Time { + if ft == 0 { + return time.Time{} + } + return time.Unix(0, (int64(ft)-epochDelta)*100) +} + +func FiletimeFromTime(t time.Time) Filetime { + if t.IsZero() { + return 0 + } + return Filetime(t.UnixNano()/100 + epochDelta) +} + +func (ft Filetime) String() string { + return ft.Time().String() +} + +func (ft Filetime) toSyscallFiletime() syscall.Filetime { + return syscall.NsecToFiletime((int64(ft) - epochDelta) * 100) +} + +// A FileInfo specifies information about a file. +type FileInfo struct { + FileID uint64 // ignored on write + Size int64 + Attributes uint32 + CreationTime Filetime + LastWriteTime Filetime + ChangeTime Filetime + LastAccessTime Filetime + SecurityDescriptor []byte + ExtendedAttributes []byte + ReparseData []byte +} + +type OpError struct { + Cim string + Op string + Err error +} + +func (e *OpError) Error() string { + s := "cim " + e.Op + " " + e.Cim + s += ": " + e.Err.Error() + return s +} + +// PathError is the error type returned by most functions in this package. +type PathError struct { + Cim string + Op string + Path string + Err error +} + +func (e *PathError) Error() string { + s := "cim " + e.Op + " " + e.Cim + s += ":" + e.Path + s += ": " + e.Err.Error() + return s +} + +type StreamError struct { + Cim string + Op string + Path string + Stream string + Err error +} + +func (e *StreamError) Error() string { + s := "cim " + e.Op + " " + e.Cim + s += ":" + e.Path + s += ":" + e.Stream + s += ": " + e.Err.Error() + return s +} + +type LinkError struct { + Cim string + Op string + Old string + New string + Err error +} + +func (e *LinkError) Error() string { + return "cim " + e.Op + " " + e.Old + " " + e.New + ": " + e.Err.Error() +} diff --git a/internal/cimfs/format/format.go b/internal/cimfs/format/format.go new file mode 100644 index 0000000000..4052f642ca --- /dev/null +++ b/internal/cimfs/format/format.go @@ -0,0 +1,260 @@ +package format + +import "github.com/Microsoft/go-winio/pkg/guid" + +// PageSize is the alignment of data for large files inside a CIM. +const PageSize = 4096 + +// RegionOffset encodes an offset to objects as index of the region file +// containing the object and the byte offset within that file. +type RegionOffset uint64 + +func (o RegionOffset) ByteOffset() uint64 { + return uint64(o & 0xffffffffffff) +} + +func (o RegionOffset) RegionIndex() uint16 { + return uint16(o >> 48) +} + +func NewRegionOffset(off int64, index uint16) RegionOffset { + return RegionOffset(uint64(index)<<48 | uint64(off)) +} + +// NullOffset indicates that the specified object does not exist. +const NullOffset = RegionOffset(0) + +// Magic specifies the magic number at the beginning of a file. +type Magic [8]uint8 + +var MagicValue = Magic([8]uint8{'c', 'i', 'm', 'f', 'i', 'l', 'e', '0'}) + +type Version struct { + Major, Minor uint32 +} + +var CurrentVersion = Version{2, 0} + +type FileType uint8 + +const ( + FtImage FileType = iota + FtRegion + FtObjectID +) + +// CommonHeader is the common header for all CIM-related files. +type CommonHeader struct { + Magic Magic + HeaderLength uint32 + Type FileType + Reserved uint8 + Reserved2 uint16 + Version Version + Reserved3 uint64 +} + +// Region file. +// +// Region files contain all the data and metadata for an image. They are +// arranged as unordered sequences of objects of varying size, and each region +// file type has its own alignment requirement. + +const RegionFileName = "region" + +// Each region file has a type, and all objects within that file are of the same +// type. +type RegionType uint8 + +const ( + // All metadata objects (files, directory data, security descriptors, etc.) + RtMetadata RegionType = 0 + // Page-aligned file data. + RtData + // 8-byte aligned file data (for small files). + RtSmallData + RtCount +) + +// RegionHeader is the header for the region file. +type RegionHeader struct { + Common CommonHeader + Index uint16 + Type RegionType + Reserved uint8 + Reserved2 uint32 +} + +// Object ID file +// +// There is an object ID file corresponding to each region file, containing IDs +// for each object that the region file contains. These IDs are not used at +// runtime but are used at write time to deduplicate objects. + +const ObjectIdFileName = "objectid" + +// ObjectIdHeader is the header for the object ID file. +type ObjectIdHeader struct { + Common CommonHeader + Index uint16 + Type RegionType + Reserved uint8 + Reserved2 uint32 + TableOffset uint32 + Count uint32 +} + +// ObjectID is the object ID itself, containing a length and a digest. +type ObjectID struct { + Length uint64 + Digest [24]uint8 +} + +// ObjectIdEntry is an entry in the object ID file. It contains the object ID +// and the byte offset into the corresponding region file. +type ObjectIdEntry struct { + ObjectID ObjectID + Offset uint64 +} + +type RegionSet struct { + ID guid.GUID + Count uint16 + Reserved uint16 + Reserved1 uint32 +} + +// FilesystemHeader is the header for a filesystem file. +// +// The filesystem file points to the filesystem object inside a region +// file and specifies regions sets. +type FilesystemHeader struct { + Common CommonHeader + Regions RegionSet + FilesystemOffset RegionOffset + Reserved uint32 + Reserved1 uint16 + ParentCount uint16 + // RegionSet ParentRegionSets[ParentCount]; +} + +const UpcaseTableLength = 0x10000 // Only characters in the BMP are upcased + +type FileID uint32 + +// Filesystem specifies a root directory and other metadata necessary to define +// a filesystem. +type Filesystem struct { + UpcaseTableOffset RegionOffset + FileTableDirectoryOffset RegionOffset + FileTableDirectoryLength uint32 + RootDirectory FileID +} + +// FileTableDirectoryEntry is the entry within a file table directory. Files are +// laid out in a series of file tables, and file tables are specified by a +// directory. The file table directory entry specifies the number of valid files +// within the table, as well as the entry size (which may grow to specify +// additional file metadata in the future). +type FileTableDirectoryEntry struct { + Offset RegionOffset + Count uint16 + EntrySize uint16 + Reserved uint32 +} + +const FilesPerTable = 1024 + +type StreamType uint16 + +const ( + StreamTypeData StreamType = iota + StreamTypeLinkTable + StreamTypePeImage +) + +// Stream describes a stream data and offset. It may point to file data, a link +// table (for directories), or a PeImage object for files that are PE images. +type Stream struct { + DataOffset RegionOffset // stream data or PeImage object + LengthAndType uint64 // 48, 8 +} + +func (s *Stream) Size() uint64 { + return (s.LengthAndType & 0xffffffffffff) +} + +func (s *Stream) Type() StreamType { + return StreamType((s.LengthAndType >> 48) & 0xff) +} + +func (s *Stream) Sparse() bool { + return s.LengthAndType&(1<<56) != 0 +} + +// PeImage is the stream data for a file that is a PE image. It provides an +// on-disk 4KB image mapping for a 512-byte aligned PE image. In this case, the +// image is aligned well on disk for image mappings, but it is discontiguous for +// ordinary reads. +type PeImage struct { + DataOffset RegionOffset + DataLength uint64 + ImageLength uint32 + MappingCount uint16 + Flags uint16 // ValidImage + // Mappings []PeImageMapping +} + +type PeImageMapping struct { + FileOffset uint32 + Delta uint32 +} + +type FileFlags uint16 + +const ( + FileFlagReadOnly FileFlags = 1 << iota + FileFlagHidden + FileFlagSystem + FileFlagArchive + FileFlagSparse +) + +// File represents a file in a file system. +type File struct { + Flags FileFlags + EaLength uint16 + ReparseTag uint32 + CreationTime uint64 + LastWriteTime uint64 + ChangeTime uint64 + LastAccessTime uint64 + DefaultStream Stream // file default data stream or LinkTable + SdOffset RegionOffset // uint16 counted gsl::byte[] + EaOffset RegionOffset // gsl::byte[] + ReparseOffset RegionOffset // uint16 counted gsl::byte[] + StreamTableOffset RegionOffset // LinkTable + ParentFileId FileID + BaseRegionIndex uint16 + Reserved uint16 +} + +const MaximumEaNameLength = 254 +const MaximumFullEaLength = 0xffff + +// Name represents a file's name +type Name struct { + Length uint16 + // Bytes []uint8 +} + +// LinkTable stores either directory entries or alternate data streams. +type LinkTable struct { + Length uint32 + LinkCount uint32 + // Values []T + // NameOffsets []uint32 +} + +const MaximumComponentNameLength = 255 +const MaximumPathLength = 32767 diff --git a/internal/cimfs/mount_cim.go b/internal/cimfs/mount_cim.go new file mode 100644 index 0000000000..0d5c6e958d --- /dev/null +++ b/internal/cimfs/mount_cim.go @@ -0,0 +1,83 @@ +package cimfs + +import ( + "fmt" + "path/filepath" + + "github.com/Microsoft/go-winio/pkg/guid" + hcsschema "github.com/Microsoft/hcsshim/internal/schema2" + "github.com/pkg/errors" +) + +type MountError struct { + Cim string + Op string + VolumeGUID guid.GUID + Err error +} + +func (e *MountError) Error() string { + s := "cim " + e.Op + if e.Cim != "" { + s += " " + e.Cim + } + s += " " + e.VolumeGUID.String() + ": " + e.Err.Error() + return s +} + +type cimInfo struct { + // path to the cim + path string + // Unique GUID assigned to a cim. + cimID guid.GUID + // ref count for number of times this cim was mounted. + refCount uint32 +} + +// map for information about cims mounted on the host +var hostCimMounts = make(map[string]*cimInfo) + +// Mount mounts the cim at path `cimPath` and returns the mount location of that cim. +// If this cim is already mounted then nothing is done. +func Mount(cimPath string) (string, error) { + if _, ok := hostCimMounts[cimPath]; !ok { + layerGUID, err := guid.NewV4() + if err != nil { + return "", &MountError{Cim: cimPath, Op: "Mount", Err: err} + } + if err := cimMountImage(filepath.Dir(cimPath), filepath.Base(cimPath), hcsschema.CimMountFlagCacheFiles, &layerGUID); err != nil { + return "", &MountError{Cim: cimPath, Op: "Mount", VolumeGUID: layerGUID, Err: err} + } + hostCimMounts[cimPath] = &cimInfo{cimPath, layerGUID, 0} + } + ci := hostCimMounts[cimPath] + ci.refCount += 1 + return fmt.Sprintf("\\\\?\\Volume{%s}\\", ci.cimID), nil +} + +// Returns the path ("\\?\Volume{GUID}" format) at which the cim with given cimPath is mounted +// Throws an error if the given cim is not mounted. +func GetCimMountPath(cimPath string) (string, error) { + ci, ok := hostCimMounts[cimPath] + if !ok { + return "", errors.Errorf("cim %s is not mounted", cimPath) + } + return fmt.Sprintf("\\\\?\\Volume{%s}\\", ci.cimID), nil +} + +// UnMount unmounts the cim at path `cimPath` if this is the last reference to it. +func UnMount(cimPath string) error { + ci, ok := hostCimMounts[cimPath] + if !ok { + return errors.Errorf("cim not mounted") + } + if ci.refCount == 1 { + if err := cimDismountImage(&ci.cimID); err != nil { + return &MountError{Cim: cimPath, Op: "UnMount", Err: err} + } + delete(hostCimMounts, cimPath) + } else { + ci.refCount -= 1 + } + return nil +} diff --git a/internal/cimfs/syscall_windows.go b/internal/cimfs/syscall_windows.go new file mode 100644 index 0000000000..f735b2fa8a --- /dev/null +++ b/internal/cimfs/syscall_windows.go @@ -0,0 +1,48 @@ +package cimfs + +import ( + "syscall" + "unsafe" + + "github.com/Microsoft/go-winio/pkg/guid" +) + +type g = guid.GUID +type orHKey uintptr +type FsHandle uintptr +type StreamHandle uintptr + +type cimFsFileMetadata struct { + Attributes uint32 + FileSize int64 + + CreationTime syscall.Filetime + LastWriteTime syscall.Filetime + ChangeTime syscall.Filetime + LastAccessTime syscall.Filetime + + SecurityDescriptorBuffer unsafe.Pointer + SecurityDescriptorSize uint32 + + ReparseDataBuffer unsafe.Pointer + ReparseDataSize uint32 + + ExtendedAttributes unsafe.Pointer + EACount uint32 +} + +//go:generate go run ../../mksyscall_windows.go -output zsyscall_windows.go syscall_windows.go + +//sys cimMountImage(imagePath string, fsName string, flags uint32, volumeID *g) (hr error) = cimfs.CimMountImage +//sys cimDismountImage(volumeID *g) (hr error) = cimfs.CimDismountImage + +//sys cimCreateImage(imagePath string, oldFSName *uint16, newFSName *uint16, cimFSHandle *FsHandle) (hr error) = cimfs.CimCreateImage +//sys cimCloseImage(cimFSHandle FsHandle) (hr error) = cimfs.CimCloseImage +//sys cimCommitImage(cimFSHandle FsHandle) (hr error) = cimfs.CimCommitImage + +//sys cimCreateFile(cimFSHandle FsHandle, path string, file *cimFsFileMetadata, cimStreamHandle *StreamHandle) (hr error) = cimfs.CimCreateFile +//sys cimCloseStream(cimStreamHandle StreamHandle) (hr error) = cimfs.CimCloseStream +//sys cimWriteStream(cimStreamHandle StreamHandle, buffer uintptr, bufferSize uint32) (hr error) = cimfs.CimWriteStream +//sys cimDeletePath(cimFSHandle FsHandle, path string) (hr error) = cimfs.CimDeletePath +//sys cimCreateHardLink(cimFSHandle FsHandle, newPath string, oldPath string) (hr error) = cimfs.CimCreateHardLink +//sys cimCreateAlternateStream(cimFSHandle FsHandle, path string, size uint64, cimStreamHandle *StreamHandle) (hr error) = cimfs.CimCreateAlternateStream diff --git a/internal/cimfs/zsyscall_windows.go b/internal/cimfs/zsyscall_windows.go new file mode 100644 index 0000000000..690f4cc65f --- /dev/null +++ b/internal/cimfs/zsyscall_windows.go @@ -0,0 +1,238 @@ +// Code generated mksyscall_windows.exe DO NOT EDIT + +package cimfs + +import ( + "syscall" + "unsafe" + + "golang.org/x/sys/windows" +) + +var _ unsafe.Pointer + +// Do the interface allocations only once for common +// Errno values. +const ( + errnoERROR_IO_PENDING = 997 +) + +var ( + errERROR_IO_PENDING error = syscall.Errno(errnoERROR_IO_PENDING) +) + +// errnoErr returns common boxed Errno values, to prevent +// allocations at runtime. +func errnoErr(e syscall.Errno) error { + switch e { + case 0: + return nil + case errnoERROR_IO_PENDING: + return errERROR_IO_PENDING + } + // TODO: add more here, after collecting data on the common + // error values see on Windows. (perhaps when running + // all.bat?) + return e +} + +var ( + modcimfs = windows.NewLazySystemDLL("cimfs.dll") + + procCimMountImage = modcimfs.NewProc("CimMountImage") + procCimDismountImage = modcimfs.NewProc("CimDismountImage") + procCimCreateImage = modcimfs.NewProc("CimCreateImage") + procCimCloseImage = modcimfs.NewProc("CimCloseImage") + procCimCommitImage = modcimfs.NewProc("CimCommitImage") + procCimCreateFile = modcimfs.NewProc("CimCreateFile") + procCimCloseStream = modcimfs.NewProc("CimCloseStream") + procCimWriteStream = modcimfs.NewProc("CimWriteStream") + procCimDeletePath = modcimfs.NewProc("CimDeletePath") + procCimCreateHardLink = modcimfs.NewProc("CimCreateHardLink") + procCimCreateAlternateStream = modcimfs.NewProc("CimCreateAlternateStream") +) + +func cimMountImage(imagePath string, fsName string, flags uint32, volumeID *g) (hr error) { + var _p0 *uint16 + _p0, hr = syscall.UTF16PtrFromString(imagePath) + if hr != nil { + return + } + var _p1 *uint16 + _p1, hr = syscall.UTF16PtrFromString(fsName) + if hr != nil { + return + } + return _cimMountImage(_p0, _p1, flags, volumeID) +} + +func _cimMountImage(imagePath *uint16, fsName *uint16, flags uint32, volumeID *g) (hr error) { + r0, _, _ := syscall.Syscall6(procCimMountImage.Addr(), 4, uintptr(unsafe.Pointer(imagePath)), uintptr(unsafe.Pointer(fsName)), uintptr(flags), uintptr(unsafe.Pointer(volumeID)), 0, 0) + if int32(r0) < 0 { + if r0&0x1fff0000 == 0x00070000 { + r0 &= 0xffff + } + hr = syscall.Errno(r0) + } + return +} + +func cimDismountImage(volumeID *g) (hr error) { + r0, _, _ := syscall.Syscall(procCimDismountImage.Addr(), 1, uintptr(unsafe.Pointer(volumeID)), 0, 0) + if int32(r0) < 0 { + if r0&0x1fff0000 == 0x00070000 { + r0 &= 0xffff + } + hr = syscall.Errno(r0) + } + return +} + +func cimCreateImage(imagePath string, oldFSName *uint16, newFSName *uint16, cimFSHandle *FsHandle) (hr error) { + var _p0 *uint16 + _p0, hr = syscall.UTF16PtrFromString(imagePath) + if hr != nil { + return + } + return _cimCreateImage(_p0, oldFSName, newFSName, cimFSHandle) +} + +func _cimCreateImage(imagePath *uint16, oldFSName *uint16, newFSName *uint16, cimFSHandle *FsHandle) (hr error) { + r0, _, _ := syscall.Syscall6(procCimCreateImage.Addr(), 4, uintptr(unsafe.Pointer(imagePath)), uintptr(unsafe.Pointer(oldFSName)), uintptr(unsafe.Pointer(newFSName)), uintptr(unsafe.Pointer(cimFSHandle)), 0, 0) + if int32(r0) < 0 { + if r0&0x1fff0000 == 0x00070000 { + r0 &= 0xffff + } + hr = syscall.Errno(r0) + } + return +} + +func cimCloseImage(cimFSHandle FsHandle) (hr error) { + r0, _, _ := syscall.Syscall(procCimCloseImage.Addr(), 1, uintptr(cimFSHandle), 0, 0) + if int32(r0) < 0 { + if r0&0x1fff0000 == 0x00070000 { + r0 &= 0xffff + } + hr = syscall.Errno(r0) + } + return +} + +func cimCommitImage(cimFSHandle FsHandle) (hr error) { + r0, _, _ := syscall.Syscall(procCimCommitImage.Addr(), 1, uintptr(cimFSHandle), 0, 0) + if int32(r0) < 0 { + if r0&0x1fff0000 == 0x00070000 { + r0 &= 0xffff + } + hr = syscall.Errno(r0) + } + return +} + +func cimCreateFile(cimFSHandle FsHandle, path string, file *cimFsFileMetadata, cimStreamHandle *StreamHandle) (hr error) { + var _p0 *uint16 + _p0, hr = syscall.UTF16PtrFromString(path) + if hr != nil { + return + } + return _cimCreateFile(cimFSHandle, _p0, file, cimStreamHandle) +} + +func _cimCreateFile(cimFSHandle FsHandle, path *uint16, file *cimFsFileMetadata, cimStreamHandle *StreamHandle) (hr error) { + r0, _, _ := syscall.Syscall6(procCimCreateFile.Addr(), 4, uintptr(cimFSHandle), uintptr(unsafe.Pointer(path)), uintptr(unsafe.Pointer(file)), uintptr(unsafe.Pointer(cimStreamHandle)), 0, 0) + if int32(r0) < 0 { + if r0&0x1fff0000 == 0x00070000 { + r0 &= 0xffff + } + hr = syscall.Errno(r0) + } + return +} + +func cimCloseStream(cimStreamHandle StreamHandle) (hr error) { + r0, _, _ := syscall.Syscall(procCimCloseStream.Addr(), 1, uintptr(cimStreamHandle), 0, 0) + if int32(r0) < 0 { + if r0&0x1fff0000 == 0x00070000 { + r0 &= 0xffff + } + hr = syscall.Errno(r0) + } + return +} + +func cimWriteStream(cimStreamHandle StreamHandle, buffer uintptr, bufferSize uint32) (hr error) { + r0, _, _ := syscall.Syscall(procCimWriteStream.Addr(), 3, uintptr(cimStreamHandle), uintptr(buffer), uintptr(bufferSize)) + if int32(r0) < 0 { + if r0&0x1fff0000 == 0x00070000 { + r0 &= 0xffff + } + hr = syscall.Errno(r0) + } + return +} + +func cimDeletePath(cimFSHandle FsHandle, path string) (hr error) { + var _p0 *uint16 + _p0, hr = syscall.UTF16PtrFromString(path) + if hr != nil { + return + } + return _cimDeletePath(cimFSHandle, _p0) +} + +func _cimDeletePath(cimFSHandle FsHandle, path *uint16) (hr error) { + r0, _, _ := syscall.Syscall(procCimDeletePath.Addr(), 2, uintptr(cimFSHandle), uintptr(unsafe.Pointer(path)), 0) + if int32(r0) < 0 { + if r0&0x1fff0000 == 0x00070000 { + r0 &= 0xffff + } + hr = syscall.Errno(r0) + } + return +} + +func cimCreateHardLink(cimFSHandle FsHandle, newPath string, oldPath string) (hr error) { + var _p0 *uint16 + _p0, hr = syscall.UTF16PtrFromString(newPath) + if hr != nil { + return + } + var _p1 *uint16 + _p1, hr = syscall.UTF16PtrFromString(oldPath) + if hr != nil { + return + } + return _cimCreateHardLink(cimFSHandle, _p0, _p1) +} + +func _cimCreateHardLink(cimFSHandle FsHandle, newPath *uint16, oldPath *uint16) (hr error) { + r0, _, _ := syscall.Syscall(procCimCreateHardLink.Addr(), 3, uintptr(cimFSHandle), uintptr(unsafe.Pointer(newPath)), uintptr(unsafe.Pointer(oldPath))) + if int32(r0) < 0 { + if r0&0x1fff0000 == 0x00070000 { + r0 &= 0xffff + } + hr = syscall.Errno(r0) + } + return +} + +func cimCreateAlternateStream(cimFSHandle FsHandle, path string, size uint64, cimStreamHandle *StreamHandle) (hr error) { + var _p0 *uint16 + _p0, hr = syscall.UTF16PtrFromString(path) + if hr != nil { + return + } + return _cimCreateAlternateStream(cimFSHandle, _p0, size, cimStreamHandle) +} + +func _cimCreateAlternateStream(cimFSHandle FsHandle, path *uint16, size uint64, cimStreamHandle *StreamHandle) (hr error) { + r0, _, _ := syscall.Syscall6(procCimCreateAlternateStream.Addr(), 4, uintptr(cimFSHandle), uintptr(unsafe.Pointer(path)), uintptr(size), uintptr(unsafe.Pointer(cimStreamHandle)), 0, 0) + if int32(r0) < 0 { + if r0&0x1fff0000 == 0x00070000 { + r0 &= 0xffff + } + hr = syscall.Errno(r0) + } + return +} diff --git a/internal/schema2/cim_mount.go b/internal/schema2/cim_mount.go new file mode 100644 index 0000000000..ab70e94767 --- /dev/null +++ b/internal/schema2/cim_mount.go @@ -0,0 +1,24 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.5 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +const ( + CimMountFlagNone uint32 = 0x0 + CimMountFlagChildOnly uint32 = 0x1 + CimMountFlagEnableDax uint32 = 0x2 + CimMountFlagCacheFiles uint32 = 0x4 +) + +type CimMount struct { + ImagePath string `json:"ImagePath,omitempty"` + FileSystemName string `json:"FileSystemName,omitempty"` + VolumeGuid string `json:"VolumeGuid,omitempty"` + MountFlags uint32 `json:"MountFlags,omitempty"` +} diff --git a/internal/winapi/utils.go b/internal/winapi/utils.go index f3055d4175..fb895f2043 100644 --- a/internal/winapi/utils.go +++ b/internal/winapi/utils.go @@ -1,7 +1,9 @@ package winapi import ( + "encoding/binary" "errors" + "strings" "syscall" "unicode/utf16" "unsafe" @@ -13,6 +15,24 @@ type UnicodeString struct { Buffer *uint16 } +// parseUtf16LE parses a UTF-16LE byte array into a string (without passing +// through a uint16 or rune array). +func ParseUtf16LE(b []byte) string { + var sb strings.Builder + sb.Grow(len(b) / 2) + for len(b) > 0 { + r := rune(binary.LittleEndian.Uint16(b)) + if utf16.IsSurrogate(r) && len(b) > 2 { + sb.WriteRune(utf16.DecodeRune(r, rune(binary.LittleEndian.Uint16(b[2:])))) + b = b[4:] + } else { + sb.WriteRune(r) + b = b[2:] + } + } + return sb.String() +} + //String converts a UnicodeString to a golang string func (uni UnicodeString) String() string { p := (*[0xffff]uint16)(unsafe.Pointer(uni.Buffer)) diff --git a/osversion/windowsbuilds.go b/osversion/windowsbuilds.go index e9267b9554..765dff3699 100644 --- a/osversion/windowsbuilds.go +++ b/osversion/windowsbuilds.go @@ -35,4 +35,8 @@ const ( // V20H2 corresponds to Windows Server 20H2 (semi-annual channel). V20H2 = 19042 + + // minimum build required for cimfs + //TODO(ambarve): replace this with correct value before merging + IRON_BUILD = 21280 )