Improve DLL loading behavior on Windows

UI/obs-app.cpp

@@ -2447,6 +2447,14 @@ static bool vc_runtime_outdated()
 
 	return true;
 }
+
+static void set_process_mitigation_policies()
+{
+	/* DLL planting protection - prefer system32 images  */
+	PROCESS_MITIGATION_IMAGE_LOAD_POLICY policy = {};
+	policy.PreferSystem32Images = 1;
+	SetProcessMitigationPolicy(ProcessImageLoadPolicy, &policy, sizeof(policy));
+}
 #endif
 
 int main(int argc, char *argv[])
@@ -2478,11 +2486,17 @@ int main(int argc, char *argv[])
 	// Abort as early as possible if MSVC runtime is outdated
 	if (vc_runtime_outdated())
 		return 1;
+
 	// Try to keep this as early as possible
 	install_dll_blocklist_hook();
 
+	set_process_mitigation_policies();
+
 	obs_init_win32_crash_handler();
 	SetErrorMode(SEM_FAILCRITICALERRORS);
+	SetSearchPathMode(BASE_SEARCH_PATH_ENABLE_SAFE_SEARCHMODE | BASE_SEARCH_PATH_PERMANENT);
+	SetDefaultDllDirectories(LOAD_LIBRARY_SEARCH_DEFAULT_DIRS);
+	SetDllDirectoryW(L"");
 	load_debug_privilege();
 	base_set_crash_handler(main_crash_handler, nullptr);
 

libobs/util/platform-windows.c

@@ -84,19 +84,21 @@ void *os_dlopen(const char *path)
 	 * dynamically loaded libraries on windows to search for dependent
 	 * libraries that are within the library's own directory */
 	wpath_slash = wcsrchr(wpath, L'/');
+
 	if (wpath_slash) {
-		*wpath_slash = 0;
-		SetDllDirectoryW(wpath);
-		*wpath_slash = L'/';
-	}
+		wchar_t fullpath[MAX_PATH];
 
-	h_library = LoadLibraryW(wpath);
+		/* FIXME: this should use the OBS install dir as a base and not rely on the current directory */
+		if (GetFullPathNameW(wpath, MAX_PATH, fullpath, NULL)) {
+			h_library = LoadLibraryExW(fullpath, NULL,
+						   LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR | LOAD_LIBRARY_SEARCH_DEFAULT_DIRS);
+		}
+	} else {
+		h_library = LoadLibraryExW(wpath, NULL, LOAD_LIBRARY_SEARCH_DEFAULT_DIRS);
+	}
 
 	bfree(wpath);
 
-	if (wpath_slash)
-		SetDllDirectoryW(NULL);
-
 	if (!h_library) {
 		DWORD error = GetLastError();
 

plugins/coreaudio-encoder/windows-imports.h

@@ -382,12 +382,10 @@ static bool load_from_shell_path(REFKNOWNFOLDERID rfid, const wchar_t *subpath)
 	}
 
 	wchar_t path[MAX_PATH];
-	_snwprintf(path, MAX_PATH, L"%s\\%s", sh_path, subpath);
+	_snwprintf(path, MAX_PATH, L"%s\\%s\\CoreAudioToolbox.dll", sh_path, subpath);
 	CoTaskMemFree(sh_path);
 
-	SetDllDirectory(path);
-	audio_toolbox = LoadLibraryW(L"CoreAudioToolbox.dll");
-	SetDllDirectory(nullptr);
+	audio_toolbox = LoadLibraryExW(path, NULL, LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR | LOAD_LIBRARY_SEARCH_DEFAULT_DIRS);
 
 	return !!audio_toolbox;
 }
@@ -397,7 +395,7 @@ static bool load_lib(void)
 	/* -------------------------------------------- */
 	/* attempt to load from path                    */
 
-	audio_toolbox = LoadLibraryW(L"CoreAudioToolbox.dll");
+	audio_toolbox = LoadLibraryExW(L"CoreAudioToolbox.dll", NULL, LOAD_LIBRARY_SAFE_CURRENT_DIRS);
 	if (!!audio_toolbox)
 		return true;
 

plugins/obs-ffmpeg/ffmpeg-mux/ffmpeg-mux.c

@@ -1146,6 +1146,13 @@ int main(int argc, char *argv[])
 	char **argv;
 
 	SetErrorMode(SEM_FAILCRITICALERRORS);
+	SetSearchPathMode(BASE_SEARCH_PATH_ENABLE_SAFE_SEARCHMODE | BASE_SEARCH_PATH_PERMANENT);
+	SetDefaultDllDirectories(LOAD_LIBRARY_SEARCH_DEFAULT_DIRS);
+	SetDllDirectoryW(L"");
+
+	PROCESS_MITIGATION_IMAGE_LOAD_POLICY policy = {0};
+	policy.PreferSystem32Images = 1;
+	SetProcessMitigationPolicy(ProcessImageLoadPolicy, &policy, sizeof(policy));
 
 	argv = malloc(argc * sizeof(char *));
 	for (int i = 0; i < argc; i++) {

plugins/win-capture/get-graphics-offsets/get-graphics-offsets.c

@@ -17,6 +17,13 @@ int main(int argc, char *argv[])
 	wc.lpszClassName = DUMMY_WNDCLASS;
 
 	SetErrorMode(SEM_FAILCRITICALERRORS);
+	SetSearchPathMode(BASE_SEARCH_PATH_ENABLE_SAFE_SEARCHMODE | BASE_SEARCH_PATH_PERMANENT);
+	SetDefaultDllDirectories(LOAD_LIBRARY_SEARCH_DEFAULT_DIRS);
+	SetDllDirectoryW(L"");
+
+	PROCESS_MITIGATION_IMAGE_LOAD_POLICY policy = {0};
+	policy.PreferSystem32Images = 1;
+	SetProcessMitigationPolicy(ProcessImageLoadPolicy, &policy, sizeof(policy));
 
 	if (!RegisterClassA(&wc)) {
 		printf("failed to register '%s'\n", DUMMY_WNDCLASS);

plugins/win-capture/inject-helper/inject-helper.c

@@ -97,6 +97,14 @@ int main(void)
 	int ret = INJECT_ERROR_INVALID_PARAMS;
 
 	SetErrorMode(SEM_FAILCRITICALERRORS);
+	SetSearchPathMode(BASE_SEARCH_PATH_ENABLE_SAFE_SEARCHMODE | BASE_SEARCH_PATH_PERMANENT);
+	SetDefaultDllDirectories(LOAD_LIBRARY_SEARCH_DEFAULT_DIRS);
+	SetDllDirectoryW(L"");
+
+	PROCESS_MITIGATION_IMAGE_LOAD_POLICY policy = {0};
+	policy.PreferSystem32Images = 1;
+	SetProcessMitigationPolicy(ProcessImageLoadPolicy, &policy, sizeof(policy));
+
 	load_debug_privilege();
 
 	pCommandLineW = GetCommandLineW();