FileDialog doesnt work in CoreRT build

[CoffeeParser]

Hi, none of the filedialogs seem to be working in the CoreRT build.
Following the advise in dotnet/corert#6252 (comment) I managed to implement a basic FileDialog in the Core2D CoreRT and CoreCLR builds (works in both).

The problem with CoreRT and COM Objects: CoreRT doesnt call CoInitializeEx and CoUninitialize. CoreCLR does call both Methods and a user doesnt have to take care... In order to get FileDialogs working in the Core2D CoreRT build I had to call CoInitializeEx with COINIT_MULTITHREADED parameter inside a separate Task! Calling CoInitializeEx on the main thread with COINIT_APARTMENTTHREADED always lead to a Fail Fast Exception deep inside thread.cpp of CoreRT. Might be worth checking if avalonia does call CoInitializeEx somewhere else in the codebase, I didn't check that.

Anyway here's the code to open a file dialog in both CoreCLR and CoreRT of Core2D.

using System;
using System.Runtime.InteropServices;

namespace Core2D.UI.Avalonia.Editor
{
    public enum DialogResult
    {
        Cancel, Abort, OK
    }

    public class FolderBrowser
    {
        public string DirectoryPath { get; set; }

        public DialogResult ShowDialog(IntPtr? owner)
        {
            IntPtr hwndOwner = IntPtr.Zero; //owner ?? GetActiveWindow(); // doesnt work here

            using (var dialog = FileOpenDialog.Create())
            {
                if (!string.IsNullOrEmpty(DirectoryPath))
                {
                    IntPtr idl;
                    uint atts = 0;
                    if (SHILCreateFromPath(DirectoryPath, out idl, ref atts) == 0)
                    {
                        using (var item = new ShellItem())
                        {
                            if (SHCreateShellItem(IntPtr.Zero, IntPtr.Zero, idl, out item.Pointer) == 0)
                            {
                                dialog.SetFolder(item);
                            }
                        }
                        Marshal.FreeCoTaskMem(idl);
                    }
                }
                dialog.SetOptions(FOS.FOS_PICKFOLDERS | FOS.FOS_FORCEFILESYSTEM);
                uint hr = dialog.Show(hwndOwner);
                if (hr == ERROR_CANCELLED)
                {
                    // thats the important call for CoreRT
                    CoUninitialize();
                    return DialogResult.Cancel;
                }

                if (hr != 0)
                {
                    // thats the important call for CoreRT
                    CoUninitialize();
                    return DialogResult.Abort;
                }
                    

                using (var item = dialog.GetResult())
                {
                    string path;
                    item.GetDisplayName(SIGDN.SIGDN_FILESYSPATH, out path);
                    DirectoryPath = path;
                }
                // thats the important call for CoreRT
                CoUninitialize();
                return DialogResult.OK;
            }
            
        }

        [DllImport("ole32.dll")]
        private static extern int CoInitializeEx(IntPtr pvReserved, COINIT dwCoInit);

        [DllImport("ole32.dll")]
        private static extern void CoUninitialize();

        [DllImport("shell32.dll")]
        private static extern int SHILCreateFromPath([MarshalAs(UnmanagedType.LPWStr)] string pszPath, out IntPtr ppIdl, ref uint rgflnOut);

        [DllImport("shell32.dll")]
        private static extern int SHCreateShellItem(IntPtr pidlParent, IntPtr psfParent, IntPtr pidl, out IntPtr ppsi);

        [DllImport("user32.dll")]
        private static extern IntPtr GetActiveWindow();

        private const uint ERROR_CANCELLED = 0x800704C7;

        public unsafe class MiniComObject : IDisposable
        {
            public IntPtr Pointer;

            delegate int IUnknown_Release(IntPtr thisPtr);
            public void Dispose()
            {
                Marshal.GetDelegateForFunctionPointer<IUnknown_Release>(*((*(IntPtr**)Pointer) + 2))(Pointer);
                Pointer = IntPtr.Zero;
            }
        }

        public unsafe class FileOpenDialog : MiniComObject
        {
            [DllImport("ole32.dll")]
            private static extern int CoCreateInstance(ref Guid rclsid, IntPtr pUnkOuter,
                                       Int32 dwClsContext,
                                       ref Guid riid,
                                       out IntPtr ppv);

            public static FileOpenDialog Create()
            {
                // thats the important call for CoreRT
                Marshal.ThrowExceptionForHR(CoInitializeEx(IntPtr.Zero, COINIT.COINIT_MULTITHREADED), new IntPtr(-1));

                FileOpenDialog result = new FileOpenDialog();
                Guid CLSID_FileOpenDialog = new Guid("DC1C5A9C-E88A-4dde-A5A1-60F82A20AEF7");
                Guid IID_IFileOpenDialog = new Guid("42f85136-db7e-439c-85f1-e4075d135fc8");
                int hr = CoCreateInstance(ref CLSID_FileOpenDialog, IntPtr.Zero,
                                       1, // CLSCTX_INPROC_SERVER,
                                       ref IID_IFileOpenDialog,
                                       out result.Pointer);
                Marshal.ThrowExceptionForHR(hr, new IntPtr(-1));

                return result;
            }

            delegate uint IFileOpenDialog_Show(IntPtr thisPtr, IntPtr parent);
            public uint Show([In] IntPtr parent)
            {
                return Marshal.GetDelegateForFunctionPointer<IFileOpenDialog_Show>(*((*(IntPtr**)Pointer) + 3))(Pointer, parent);
            }

            delegate int IFileOpenDialog_SetOptions(IntPtr thisPtr, FOS fos);
            public void SetOptions(FOS fos)
            {
                Marshal.ThrowExceptionForHR(Marshal.GetDelegateForFunctionPointer<IFileOpenDialog_SetOptions>(*((*(IntPtr**)Pointer) + 9))(Pointer, fos), new IntPtr(-1));
            }

            delegate int IFileOpenDialog_SetFolder(IntPtr thisPtr, IntPtr psi);
            public void SetFolder(ShellItem psi)
            {
                Marshal.ThrowExceptionForHR(Marshal.GetDelegateForFunctionPointer<IFileOpenDialog_SetFolder>(*((*(IntPtr**)Pointer) + 12))(Pointer, psi.Pointer), new IntPtr(-1));
            }

            delegate int IFileOpenDialog_GetResult(IntPtr thisPtr, out IntPtr ppsi);
            public ShellItem GetResult()
            {
                var result = new ShellItem();
                Marshal.ThrowExceptionForHR(Marshal.GetDelegateForFunctionPointer<IFileOpenDialog_GetResult>(*((*(IntPtr**)Pointer) + 20))(Pointer, out result.Pointer), new IntPtr(-1));
                return result;
            }
        }

        public unsafe class ShellItem : MiniComObject
        {
            delegate int IShellItem_GetDisplayName(IntPtr thisPtr, SIGDN sigdnName, [MarshalAs(UnmanagedType.LPWStr)] out string ppszName);
            public void GetDisplayName(SIGDN sigdnName, out string ppszName)
            {
                Marshal.ThrowExceptionForHR(Marshal.GetDelegateForFunctionPointer<IShellItem_GetDisplayName>(*((*(IntPtr**)Pointer) + 5))(Pointer, sigdnName, out ppszName), new IntPtr(-1));
            }
        }

#if false // just left in here for manual pointer arithmetic: baseAdress+3 = first method of class, see IFileOpenDialog_Show
        [ComImport]
        [Guid("42f85136-db7e-439c-85f1-e4075d135fc8")]
        [InterfaceType(ComInterfaceType.InterfaceIsIUnknown)]
        public interface IFileOpenDialog
        {
            [PreserveSig]
            uint Show([In] IntPtr parent); // IModalWindow
            void SetFileTypes();  // not fully defined
            void SetFileTypeIndex([In] uint iFileType);
            void GetFileTypeIndex(out uint piFileType);
            void Advise(); // not fully defined
            void Unadvise();
            void SetOptions([In] FOS fos);
            void GetOptions(out FOS pfos);
            void SetDefaultFolder(IShellItem psi);
            void SetFolder(IShellItem psi);
            void GetFolder(out IShellItem ppsi);
            void GetCurrentSelection(out IShellItem ppsi);
            void SetFileName([In, MarshalAs(UnmanagedType.LPWStr)] string pszName);
            void GetFileName([MarshalAs(UnmanagedType.LPWStr)] out string pszName);
            void SetTitle([In, MarshalAs(UnmanagedType.LPWStr)] string pszTitle);
            void SetOkButtonLabel([In, MarshalAs(UnmanagedType.LPWStr)] string pszText);
            void SetFileNameLabel([In, MarshalAs(UnmanagedType.LPWStr)] string pszLabel);
            void GetResult(out IShellItem ppsi);
            void AddPlace(IShellItem psi, int alignment);
            void SetDefaultExtension([In, MarshalAs(UnmanagedType.LPWStr)] string pszDefaultExtension);
            void Close(int hr);
            void SetClientGuid();  // not fully defined
            void ClearClientData();
            void SetFilter([MarshalAs(UnmanagedType.Interface)] IntPtr pFilter);
            void GetResults([MarshalAs(UnmanagedType.Interface)] out IntPtr ppenum); // not fully defined
            void GetSelectedItems([MarshalAs(UnmanagedType.Interface)] out IntPtr ppsai); // not fully defined
        }

        public interface IShellItem
        {
            void BindToHandler(); // not fully defined
            void GetParent(); // not fully defined
            void GetDisplayName([In] SIGDN sigdnName, [MarshalAs(UnmanagedType.LPWStr)] out string ppszName);
            void GetAttributes();  // not fully defined
            void Compare();  // not fully defined
        }
#endif

        public enum SIGDN : uint
        {
            SIGDN_DESKTOPABSOLUTEEDITING = 0x8004c000,
            SIGDN_DESKTOPABSOLUTEPARSING = 0x80028000,
            SIGDN_FILESYSPATH = 0x80058000,
            SIGDN_NORMALDISPLAY = 0,
            SIGDN_PARENTRELATIVE = 0x80080001,
            SIGDN_PARENTRELATIVEEDITING = 0x80031001,
            SIGDN_PARENTRELATIVEFORADDRESSBAR = 0x8007c001,
            SIGDN_PARENTRELATIVEPARSING = 0x80018001,
            SIGDN_URL = 0x80068000
        }

        [Flags]
        public enum FOS
        {
            FOS_ALLNONSTORAGEITEMS = 0x80,
            FOS_ALLOWMULTISELECT = 0x200,
            FOS_CREATEPROMPT = 0x2000,
            FOS_DEFAULTNOMINIMODE = 0x20000000,
            FOS_DONTADDTORECENT = 0x2000000,
            FOS_FILEMUSTEXIST = 0x1000,
            FOS_FORCEFILESYSTEM = 0x40,
            FOS_FORCESHOWHIDDEN = 0x10000000,
            FOS_HIDEMRUPLACES = 0x20000,
            FOS_HIDEPINNEDPLACES = 0x40000,
            FOS_NOCHANGEDIR = 8,
            FOS_NODEREFERENCELINKS = 0x100000,
            FOS_NOREADONLYRETURN = 0x8000,
            FOS_NOTESTFILECREATE = 0x10000,
            FOS_NOVALIDATE = 0x100,
            FOS_OVERWRITEPROMPT = 2,
            FOS_PATHMUSTEXIST = 0x800,
            FOS_PICKFOLDERS = 0x20,
            FOS_SHAREAWARE = 0x4000,
            FOS_STRICTFILETYPES = 4
        }

        [Flags]
        public enum COINIT : int //tagCOINIT
        {
            COINIT_MULTITHREADED = 0x0, //Initializes the thread for multi-threaded object concurrency.
            COINIT_APARTMENTTHREADED = 0x2, //Initializes the thread for apartment-threaded object concurrency
            COINIT_DISABLE_OLE1DDE = 0x4, //Disables DDE for OLE1 support
            COINIT_SPEED_OVER_MEMORY = 0x8, //Trade memory for speed
        }
    }
}

And thats the dummy code to call the sample from Core2D AvaloniaProjectEditorPlatform.cs

public async void OnLoadLayout()
        {
            var result = await Task.Run(() =>
            {
                var folderbrowser = new FolderBrowser();
                return folderbrowser.ShowDialog(null);
            });
}

Another note on COM invocation: right now there is no other way to invoke COM Objects other than by using pointer arithmetic. It might be worth to replace the unsafe arithmetic with IntPtr.Add calls.

Please keep in mind: thats not production code and just a proof of concept to call COM Objects from CoreRT apps, in this case file dialogs.

[wieslawsoltes]

@CoffeeParser Thank you for the code. This eventually needs to be integrated into Avalonia.

https://github.com/AvaloniaUI/Avalonia/blob/ad7f0977d030a08a958a316df34d0112402f2802/src/Windows/Avalonia.Win32/Interop/UnmanagedMethods.cs#L1374
https://github.com/AvaloniaUI/Avalonia/blob/ad7f0977d030a08a958a316df34d0112402f2802/src/Windows/Avalonia.Win32/SystemDialogImpl.cs#L19