From 194f210b049fdb9575e1330b86dea849050f006a Mon Sep 17 00:00:00 2001 From: Daniel Canter Date: Wed, 16 Mar 2022 14:00:09 -0700 Subject: [PATCH 1/2] Add Go bindflt/silo definitions This change adds a couple Go bindings for calls and constants from bindfltapi.dll as well as some silo flags for job objects. Together these allow file bindings (think bind mounts on Linux) to be performed for a specific job object. The bindings are only viewable from processes within that specific job. This change additionally adds a couple unit tests for this behavior. Signed-off-by: Daniel Canter --- internal/exec/exec_test.go | 2 +- internal/jobobject/jobobject.go | 115 +++++++++++++++++- internal/jobobject/jobobject_test.go | 69 +++++++++++ internal/winapi/bindflt.go | 20 +++ internal/winapi/jobobject.go | 1 + internal/winapi/winapi.go | 2 +- internal/winapi/zsyscall_windows.go | 30 +++-- .../hcsshim/internal/winapi/bindflt.go | 20 +++ .../hcsshim/internal/winapi/jobobject.go | 1 + .../hcsshim/internal/winapi/winapi.go | 2 +- .../internal/winapi/zsyscall_windows.go | 30 +++-- 11 files changed, 274 insertions(+), 18 deletions(-) create mode 100644 internal/winapi/bindflt.go create mode 100644 test/vendor/github.com/Microsoft/hcsshim/internal/winapi/bindflt.go diff --git a/internal/exec/exec_test.go b/internal/exec/exec_test.go index 3ad62a791e..80e5eb4012 100644 --- a/internal/exec/exec_test.go +++ b/internal/exec/exec_test.go @@ -176,7 +176,7 @@ func TestExecsWithJob(t *testing.T) { } if len(pids) != 2 { - t.Fatalf("should be two pids in job object, got: %d", len(pids)) + t.Fatalf("should be two pids in job object, got: %d. Pids: %+v", len(pids), pids) } for _, pid := range pids { diff --git a/internal/jobobject/jobobject.go b/internal/jobobject/jobobject.go index b8d586b204..3d8e27bbeb 100644 --- a/internal/jobobject/jobobject.go +++ b/internal/jobobject/jobobject.go @@ -3,7 +3,10 @@ package jobobject import ( "context" "fmt" + "os" + "path/filepath" "sync" + "sync/atomic" "unsafe" "github.com/Microsoft/hcsshim/internal/queue" @@ -24,7 +27,10 @@ import ( // the job, a queue to receive iocp notifications about the lifecycle // of the job and a mutex for synchronized handle access. type JobObject struct { - handle windows.Handle + handle windows.Handle + // All accesses to this MUST be done atomically. 1 signifies that this + // job is currently a silo. + isAppSilo uint32 mq *queue.MessageQueue handleLock sync.RWMutex } @@ -56,6 +62,7 @@ const ( var ( ErrAlreadyClosed = errors.New("the handle has already been closed") ErrNotRegistered = errors.New("job is not registered to receive notifications") + ErrNotSilo = errors.New("job is not a silo") ) // Options represents the set of configurable options when making or opening a job object. @@ -68,6 +75,9 @@ type Options struct { // `UseNTVariant` specifies if we should use the `Nt` variant of Open/CreateJobObject. // Defaults to false. UseNTVariant bool + // `Silo` specifies to promote the job to a silo. This additionally sets the flag + // JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE as it is required for the upgrade to complete. + Silo bool } // Create creates a job object. @@ -134,6 +144,16 @@ func Create(ctx context.Context, options *Options) (_ *JobObject, err error) { job.mq = mq } + if options.Silo { + // This is a required setting for upgrading to a silo. + if err := job.SetTerminateOnLastHandleClose(); err != nil { + return nil, err + } + if err := job.PromoteToSilo(); err != nil { + return nil, err + } + } + return job, nil } @@ -436,3 +456,96 @@ func (job *JobObject) QueryStorageStats() (*winapi.JOBOBJECT_BASIC_AND_IO_ACCOUN } return &info, nil } + +// ApplyFileBinding makes a file binding using the Bind Filter from target to root. If the job has +// not been upgraded to a silo this call will fail. The binding is only applied and visible for processes +// running in the job, any processes on the host or in another job will not be able to see the binding. +func (job *JobObject) ApplyFileBinding(root, target string, merged bool) error { + job.handleLock.RLock() + defer job.handleLock.RUnlock() + + if job.handle == 0 { + return ErrAlreadyClosed + } + + if !job.isSilo() { + return ErrNotSilo + } + + // The parent directory needs to exist for the bind to work. + if _, err := os.Stat(filepath.Dir(root)); os.IsNotExist(err) { + if err := os.MkdirAll(filepath.Dir(root), 0); err != nil { + return err + } + } + + rootPtr, err := windows.UTF16PtrFromString(root) + if err != nil { + return err + } + + targetPtr, err := windows.UTF16PtrFromString(target) + if err != nil { + return err + } + + flags := winapi.BINDFLT_FLAG_USE_CURRENT_SILO_MAPPING + if merged { + flags |= winapi.BINDFLT_FLAG_MERGED_BIND_MAPPING + } + + if err := winapi.BfSetupFilterEx( + flags, + job.handle, + nil, + rootPtr, + targetPtr, + nil, + 0, + ); err != nil { + return fmt.Errorf("failed to bind target %q to root %q for job object: %w", target, root, err) + } + return nil +} + +// PromoteToSilo promotes a job object to a silo. There must be no running processess +// in the job for this to succeed. If the job is already a silo this is a no-op. +func (job *JobObject) PromoteToSilo() error { + job.handleLock.RLock() + defer job.handleLock.RUnlock() + + if job.handle == 0 { + return ErrAlreadyClosed + } + + if job.isSilo() { + return nil + } + + pids, err := job.Pids() + if err != nil { + return err + } + + if len(pids) != 0 { + return fmt.Errorf("job cannot have running processes to be promoted to a silo, found %d running processes", len(pids)) + } + + _, err = windows.SetInformationJobObject( + job.handle, + winapi.JobObjectCreateSilo, + 0, + 0, + ) + if err != nil { + return fmt.Errorf("failed to promote job to silo: %w", err) + } + + atomic.StoreUint32(&job.isAppSilo, 1) + return nil +} + +// isSilo returns if the job object is a silo. +func (job *JobObject) isSilo() bool { + return atomic.LoadUint32(&job.isAppSilo) == 1 +} diff --git a/internal/jobobject/jobobject_test.go b/internal/jobobject/jobobject_test.go index 68257c790b..a4877d4bdd 100644 --- a/internal/jobobject/jobobject_test.go +++ b/internal/jobobject/jobobject_test.go @@ -2,7 +2,9 @@ package jobobject import ( "context" + "os" "os/exec" + "path/filepath" "syscall" "testing" "time" @@ -281,3 +283,70 @@ func TestVerifyPidCount(t *testing.T) { t.Fatal(err) } } + +func TestSilo(t *testing.T) { + // Test asking for a silo in the options. + options := &Options{ + Silo: true, + } + job, err := Create(context.Background(), options) + if err != nil { + t.Fatal(err) + } + defer job.Close() +} + +func TestSiloFileBinding(t *testing.T) { + // Can't use osversion as the binary needs to be manifested for it to work. + // Just stat for the bindflt dll. + if _, err := os.Stat(`C:\windows\system32\bindfltapi.dll`); err != nil { + t.Skip("Bindflt not present on RS5 or lower, skipping.") + } + // Test upgrading to a silo and binding a file only the silo can see. + options := &Options{ + Silo: true, + } + job, err := Create(context.Background(), options) + if err != nil { + t.Fatal(err) + } + defer job.Close() + + target := t.TempDir() + hostPath := filepath.Join(target, "bind-test.txt") + f, err := os.Create(hostPath) + if err != nil { + t.Fatal(err) + } + defer f.Close() + + root := t.TempDir() + siloPath := filepath.Join(root, "silo-path.txt") + if err := job.ApplyFileBinding(siloPath, hostPath, false); err != nil { + t.Fatal(err) + } + + // First check that we can't see the file on the host. + if _, err := os.Stat(siloPath); err == nil { + t.Fatalf("expected to not be able to see %q on the host", siloPath) + } + + // Now check that we can see it in the silo. Couple second timeout (ping something) so + // we can be relatively sure the process has been assigned to the job before we go to check + // on the file. Unfortunately we can't use our internal/exec package that has support for + // assigning a process to a job at creation time as it causes a cyclical import. + cmd := exec.Command("cmd", "/c", "ping", "localhost", "&&", "dir", siloPath) + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + + if err := job.Assign(uint32(cmd.Process.Pid)); err != nil { + t.Fatal(err) + } + + // Process will have an exit code of 1 if dir couldn't find the file; if we get + // no error here we should be A-OK. + if err := cmd.Wait(); err != nil { + t.Fatal(err) + } +} diff --git a/internal/winapi/bindflt.go b/internal/winapi/bindflt.go new file mode 100644 index 0000000000..ab434a75b5 --- /dev/null +++ b/internal/winapi/bindflt.go @@ -0,0 +1,20 @@ +package winapi + +const ( + BINDFLT_FLAG_READ_ONLY_MAPPING uint32 = 0x00000001 + BINDFLT_FLAG_MERGED_BIND_MAPPING uint32 = 0x00000002 + BINDFLT_FLAG_USE_CURRENT_SILO_MAPPING uint32 = 0x00000004 +) + +// HRESULT +// BfSetupFilterEx( +// _In_ ULONG Flags, +// _In_opt_ HANDLE JobHandle, +// _In_opt_ PSID Sid, +// _In_ LPCWSTR VirtualizationRootPath, +// _In_ LPCWSTR VirtualizationTargetPath, +// _In_reads_opt_( VirtualizationExceptionPathCount ) LPCWSTR* VirtualizationExceptionPaths, +// _In_opt_ ULONG VirtualizationExceptionPathCount +// ); +// +//sys BfSetupFilterEx(flags uint32, jobHandle windows.Handle, sid *windows.SID, virtRootPath *uint16, virtTargetPath *uint16, virtExceptions **uint16, virtExceptionPathCount uint32) (hr error) = bindfltapi.BfSetupFilterEx? diff --git a/internal/winapi/jobobject.go b/internal/winapi/jobobject.go index b2ca47c02b..44371b3886 100644 --- a/internal/winapi/jobobject.go +++ b/internal/winapi/jobobject.go @@ -52,6 +52,7 @@ const ( JobObjectLimitViolationInformation uint32 = 13 JobObjectMemoryUsageInformation uint32 = 28 JobObjectNotificationLimitInformation2 uint32 = 33 + JobObjectCreateSilo uint32 = 35 JobObjectIoAttribution uint32 = 42 ) diff --git a/internal/winapi/winapi.go b/internal/winapi/winapi.go index d2cc9d9fba..298ff838ac 100644 --- a/internal/winapi/winapi.go +++ b/internal/winapi/winapi.go @@ -2,4 +2,4 @@ // be thought of as an extension to golang.org/x/sys/windows. package winapi -//go:generate go run ..\..\mksyscall_windows.go -output zsyscall_windows.go user.go console.go system.go net.go path.go thread.go jobobject.go logon.go memory.go process.go processor.go devices.go filesystem.go errors.go +//go:generate go run ..\..\mksyscall_windows.go -output zsyscall_windows.go bindflt.go user.go console.go system.go net.go path.go thread.go jobobject.go logon.go memory.go process.go processor.go devices.go filesystem.go errors.go diff --git a/internal/winapi/zsyscall_windows.go b/internal/winapi/zsyscall_windows.go index fc2732a64a..935219b063 100644 --- a/internal/winapi/zsyscall_windows.go +++ b/internal/winapi/zsyscall_windows.go @@ -37,13 +37,15 @@ func errnoErr(e syscall.Errno) error { } var ( - modnetapi32 = windows.NewLazySystemDLL("netapi32.dll") - modkernel32 = windows.NewLazySystemDLL("kernel32.dll") - modntdll = windows.NewLazySystemDLL("ntdll.dll") - modiphlpapi = windows.NewLazySystemDLL("iphlpapi.dll") - modadvapi32 = windows.NewLazySystemDLL("advapi32.dll") - modcfgmgr32 = windows.NewLazySystemDLL("cfgmgr32.dll") - + modbindfltapi = windows.NewLazySystemDLL("bindfltapi.dll") + modnetapi32 = windows.NewLazySystemDLL("netapi32.dll") + modkernel32 = windows.NewLazySystemDLL("kernel32.dll") + modntdll = windows.NewLazySystemDLL("ntdll.dll") + modiphlpapi = windows.NewLazySystemDLL("iphlpapi.dll") + modadvapi32 = windows.NewLazySystemDLL("advapi32.dll") + modcfgmgr32 = windows.NewLazySystemDLL("cfgmgr32.dll") + + procBfSetupFilterEx = modbindfltapi.NewProc("BfSetupFilterEx") procNetLocalGroupGetInfo = modnetapi32.NewProc("NetLocalGroupGetInfo") procNetUserAdd = modnetapi32.NewProc("NetUserAdd") procNetUserDel = modnetapi32.NewProc("NetUserDel") @@ -77,6 +79,20 @@ var ( procRtlNtStatusToDosError = modntdll.NewProc("RtlNtStatusToDosError") ) +func BfSetupFilterEx(flags uint32, jobHandle windows.Handle, sid *windows.SID, virtRootPath *uint16, virtTargetPath *uint16, virtExceptions **uint16, virtExceptionPathCount uint32) (hr error) { + if hr = procBfSetupFilterEx.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall9(procBfSetupFilterEx.Addr(), 7, uintptr(flags), uintptr(jobHandle), uintptr(unsafe.Pointer(sid)), uintptr(unsafe.Pointer(virtRootPath)), uintptr(unsafe.Pointer(virtTargetPath)), uintptr(unsafe.Pointer(virtExceptions)), uintptr(virtExceptionPathCount), 0, 0) + if int32(r0) < 0 { + if r0&0x1fff0000 == 0x00070000 { + r0 &= 0xffff + } + hr = syscall.Errno(r0) + } + return +} + func netLocalGroupGetInfo(serverName *uint16, groupName *uint16, level uint32, bufptr **byte) (status error) { r0, _, _ := syscall.Syscall6(procNetLocalGroupGetInfo.Addr(), 4, uintptr(unsafe.Pointer(serverName)), uintptr(unsafe.Pointer(groupName)), uintptr(level), uintptr(unsafe.Pointer(bufptr)), 0, 0) if r0 != 0 { diff --git a/test/vendor/github.com/Microsoft/hcsshim/internal/winapi/bindflt.go b/test/vendor/github.com/Microsoft/hcsshim/internal/winapi/bindflt.go new file mode 100644 index 0000000000..ab434a75b5 --- /dev/null +++ b/test/vendor/github.com/Microsoft/hcsshim/internal/winapi/bindflt.go @@ -0,0 +1,20 @@ +package winapi + +const ( + BINDFLT_FLAG_READ_ONLY_MAPPING uint32 = 0x00000001 + BINDFLT_FLAG_MERGED_BIND_MAPPING uint32 = 0x00000002 + BINDFLT_FLAG_USE_CURRENT_SILO_MAPPING uint32 = 0x00000004 +) + +// HRESULT +// BfSetupFilterEx( +// _In_ ULONG Flags, +// _In_opt_ HANDLE JobHandle, +// _In_opt_ PSID Sid, +// _In_ LPCWSTR VirtualizationRootPath, +// _In_ LPCWSTR VirtualizationTargetPath, +// _In_reads_opt_( VirtualizationExceptionPathCount ) LPCWSTR* VirtualizationExceptionPaths, +// _In_opt_ ULONG VirtualizationExceptionPathCount +// ); +// +//sys BfSetupFilterEx(flags uint32, jobHandle windows.Handle, sid *windows.SID, virtRootPath *uint16, virtTargetPath *uint16, virtExceptions **uint16, virtExceptionPathCount uint32) (hr error) = bindfltapi.BfSetupFilterEx? diff --git a/test/vendor/github.com/Microsoft/hcsshim/internal/winapi/jobobject.go b/test/vendor/github.com/Microsoft/hcsshim/internal/winapi/jobobject.go index b2ca47c02b..44371b3886 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/internal/winapi/jobobject.go +++ b/test/vendor/github.com/Microsoft/hcsshim/internal/winapi/jobobject.go @@ -52,6 +52,7 @@ const ( JobObjectLimitViolationInformation uint32 = 13 JobObjectMemoryUsageInformation uint32 = 28 JobObjectNotificationLimitInformation2 uint32 = 33 + JobObjectCreateSilo uint32 = 35 JobObjectIoAttribution uint32 = 42 ) diff --git a/test/vendor/github.com/Microsoft/hcsshim/internal/winapi/winapi.go b/test/vendor/github.com/Microsoft/hcsshim/internal/winapi/winapi.go index d2cc9d9fba..298ff838ac 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/internal/winapi/winapi.go +++ b/test/vendor/github.com/Microsoft/hcsshim/internal/winapi/winapi.go @@ -2,4 +2,4 @@ // be thought of as an extension to golang.org/x/sys/windows. package winapi -//go:generate go run ..\..\mksyscall_windows.go -output zsyscall_windows.go user.go console.go system.go net.go path.go thread.go jobobject.go logon.go memory.go process.go processor.go devices.go filesystem.go errors.go +//go:generate go run ..\..\mksyscall_windows.go -output zsyscall_windows.go bindflt.go user.go console.go system.go net.go path.go thread.go jobobject.go logon.go memory.go process.go processor.go devices.go filesystem.go errors.go diff --git a/test/vendor/github.com/Microsoft/hcsshim/internal/winapi/zsyscall_windows.go b/test/vendor/github.com/Microsoft/hcsshim/internal/winapi/zsyscall_windows.go index fc2732a64a..935219b063 100644 --- a/test/vendor/github.com/Microsoft/hcsshim/internal/winapi/zsyscall_windows.go +++ b/test/vendor/github.com/Microsoft/hcsshim/internal/winapi/zsyscall_windows.go @@ -37,13 +37,15 @@ func errnoErr(e syscall.Errno) error { } var ( - modnetapi32 = windows.NewLazySystemDLL("netapi32.dll") - modkernel32 = windows.NewLazySystemDLL("kernel32.dll") - modntdll = windows.NewLazySystemDLL("ntdll.dll") - modiphlpapi = windows.NewLazySystemDLL("iphlpapi.dll") - modadvapi32 = windows.NewLazySystemDLL("advapi32.dll") - modcfgmgr32 = windows.NewLazySystemDLL("cfgmgr32.dll") - + modbindfltapi = windows.NewLazySystemDLL("bindfltapi.dll") + modnetapi32 = windows.NewLazySystemDLL("netapi32.dll") + modkernel32 = windows.NewLazySystemDLL("kernel32.dll") + modntdll = windows.NewLazySystemDLL("ntdll.dll") + modiphlpapi = windows.NewLazySystemDLL("iphlpapi.dll") + modadvapi32 = windows.NewLazySystemDLL("advapi32.dll") + modcfgmgr32 = windows.NewLazySystemDLL("cfgmgr32.dll") + + procBfSetupFilterEx = modbindfltapi.NewProc("BfSetupFilterEx") procNetLocalGroupGetInfo = modnetapi32.NewProc("NetLocalGroupGetInfo") procNetUserAdd = modnetapi32.NewProc("NetUserAdd") procNetUserDel = modnetapi32.NewProc("NetUserDel") @@ -77,6 +79,20 @@ var ( procRtlNtStatusToDosError = modntdll.NewProc("RtlNtStatusToDosError") ) +func BfSetupFilterEx(flags uint32, jobHandle windows.Handle, sid *windows.SID, virtRootPath *uint16, virtTargetPath *uint16, virtExceptions **uint16, virtExceptionPathCount uint32) (hr error) { + if hr = procBfSetupFilterEx.Find(); hr != nil { + return + } + r0, _, _ := syscall.Syscall9(procBfSetupFilterEx.Addr(), 7, uintptr(flags), uintptr(jobHandle), uintptr(unsafe.Pointer(sid)), uintptr(unsafe.Pointer(virtRootPath)), uintptr(unsafe.Pointer(virtTargetPath)), uintptr(unsafe.Pointer(virtExceptions)), uintptr(virtExceptionPathCount), 0, 0) + if int32(r0) < 0 { + if r0&0x1fff0000 == 0x00070000 { + r0 &= 0xffff + } + hr = syscall.Errno(r0) + } + return +} + func netLocalGroupGetInfo(serverName *uint16, groupName *uint16, level uint32, bufptr **byte) (status error) { r0, _, _ := syscall.Syscall6(procNetLocalGroupGetInfo.Addr(), 4, uintptr(unsafe.Pointer(serverName)), uintptr(unsafe.Pointer(groupName)), uintptr(level), uintptr(unsafe.Pointer(bufptr)), 0, 0) if r0 != 0 { From 2a9584216faba1f3e02c51514c30840b8958ea8a Mon Sep 17 00:00:00 2001 From: Daniel Canter Date: Mon, 4 Apr 2022 12:07:05 -0700 Subject: [PATCH 2/2] PR Feedback Return error on file binding if stating the directory fails. Signed-off-by: Daniel Canter --- internal/jobobject/jobobject.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/internal/jobobject/jobobject.go b/internal/jobobject/jobobject.go index 3d8e27bbeb..a0257b39cd 100644 --- a/internal/jobobject/jobobject.go +++ b/internal/jobobject/jobobject.go @@ -473,7 +473,10 @@ func (job *JobObject) ApplyFileBinding(root, target string, merged bool) error { } // The parent directory needs to exist for the bind to work. - if _, err := os.Stat(filepath.Dir(root)); os.IsNotExist(err) { + if _, err := os.Stat(filepath.Dir(root)); err != nil { + if !os.IsNotExist(err) { + return err + } if err := os.MkdirAll(filepath.Dir(root), 0); err != nil { return err }