[SYCL][L0] Implement Plugin Lifetime Management (#2942)

The patch introduces new PI API - piTearDown which is called by the SYCL RT
prior to plugin library unloading. Global variables are now released in the
level zero plugin.

Signed-off-by: Byoungro So <byoungro.so@intel.com>

Author: bso-intel

sycl/doc/GlobalObjectsInRuntime.md

@@ -88,7 +88,14 @@ constructor and destructor.
 
 ## Plugins
 
-TBD
+Plugin lifetime is managed by utilizing piPluginInit() and piTearDown().
+GlobalHandler::shutdown() will tear down all registered globals before SYCL RT
+library is unloaded. It will invoke piTearDown() and unload() for each
+plugin. piTearDown() is going to perform any necessary tear-down process at the
+plugin PI level. These two APIs allow on-demand plugin lifetime management. SYCL
+RT can control the beginning and the end of the plugin. 
+
+![](images/plugin-lifetime.jpg)
 
 ## Low-level runtimes
 

sycl/doc/PluginInterface.md

@@ -121,9 +121,22 @@ The trace shows the PI API calls made when using SYCL_PI_TRACE=-1.
 bound.)
 
 ### Plugin Unloading
-The plugins not chosen to be connected to should be unloaded.
-
-TBD - Unloading a bound plugin.
+The plugins not chosen to be connected to should be unloaded. piInitializePlugins()
+can be called to load and bound the necessary plugins. In addition, piTearDown()
+can be called when plugins are not needed any more. It notifies each
+plugin to start performing its own tear-down process such as global memory
+deallocation. In the future, piTearDown() can include any other jobs that need to
+be done before the plugin is unloaded from memory. Possibly, a
+notification of the plugin unloading to lower-level plugins can be added so that
+they can clean up their own memory [TBD].
+After piTearDown() is called, the plugin can be safely unloaded by calling unload(),
+which is going to invoke OS-specific system calls to remove the dynamic library
+from memory.
+
+Each plugin should not create global variables that require non-trivial
+destructor. Pointer variables with heap memory allocation is a good example
+to be created at the global scope. A std::vector object is not. piTearDown
+will take care of deallocation of these global variables safely.
 
 ## PI API Specification
 

sycl/doc/images/plugin-lifetime.jpg

@@ -126,5 +126,6 @@ _PI_API(piextUSMGetMemAllocInfo)
 
 _PI_API(piextKernelSetArgMemObj)
 _PI_API(piextKernelSetArgSampler)
+_PI_API(piTearDown)
 
 #undef _PI_API

sycl/include/CL/sycl/detail/pi.def

@@ -1600,6 +1600,11 @@ __SYCL_EXPORT pi_result piextUSMGetMemAllocInfo(
     pi_context context, const void *ptr, pi_mem_info param_name,
     size_t param_value_size, void *param_value, size_t *param_value_size_ret);
 
+/// API to notify that the plugin should clean up its resources.
+/// No PI calls should be made until the next piPluginInit call.
+/// \param PluginParameter placeholder for future use, currenly not used.
+__SYCL_EXPORT pi_result piTearDown(void *PluginParameter);
+
 struct _pi_plugin {
   // PI version supported by host passed to the plugin. The Plugin
   // checks and writes the appropriate Function Pointers in

sycl/include/CL/sycl/detail/pi.h

@@ -123,6 +123,13 @@ __SYCL_EXPORT void contextSetExtendedDeleter(const cl::sycl::context &constext,
 // Implementation is OS dependent.
 void *loadOsLibrary(const std::string &Library);
 
+// Function to unload the shared library
+// Implementation is OS dependent (see posix-pi.cpp and windows-pi.cpp)
+int unloadOsLibrary(void *Library);
+
+// OS agnostic function to unload the shared library
+int unloadPlugin(void *Library);
+
 // Function to get Address of a symbol defined in the shared
 // library, implementation is OS dependent.
 void *getOsLibraryFuncAddress(void *Library, const std::string &FunctionName);

sycl/include/CL/sycl/detail/pi.hpp

@@ -4475,6 +4475,11 @@ pi_result cuda_piextUSMGetMemAllocInfo(pi_context context, const void *ptr,
   return result;
 }
 
+// This API is called by Sycl RT to notify the end of the plugin lifetime.
+// TODO: add a global variable lifetime management code here (see
+// pi_level_zero.cpp for reference) Currently this is just a NOOP.
+pi_result cuda_piTearDown(void *PluginParameter) { return PI_SUCCESS; }
+
 const char SupportedVersion[] = _PI_H_VERSION_STRING;
 
 pi_result piPluginInit(pi_plugin *PluginInit) {
@@ -4610,6 +4615,7 @@ pi_result piPluginInit(pi_plugin *PluginInit) {
 
   _PI_CL(piextKernelSetArgMemObj, cuda_piextKernelSetArgMemObj)
   _PI_CL(piextKernelSetArgSampler, cuda_piextKernelSetArgSampler)
+  _PI_CL(piTearDown, cuda_piTearDown)
 
 #undef _PI_CL
 

sycl/plugins/cuda/pi_cuda.cpp

@@ -12,6 +12,7 @@
 /// \ingroup sycl_pi_level_zero
 
 #include "pi_level_zero.hpp"
+#include <CL/sycl/detail/spinlock.hpp>
 #include <algorithm>
 #include <cstdarg>
 #include <cstdio>
@@ -172,6 +173,17 @@ class ReturnHelper {
 
 } // anonymous namespace
 
+// Global variables used in PI_Level_Zero
+// Note we only create a simple pointer variables such that C++ RT won't
+// deallocate them automatically at the end of the main program.
+// The heap memory allocated for these global variables reclaimed only when
+// Sycl RT calls piTearDown().
+static std::vector<pi_platform> *PiPlatformsCache =
+    new std::vector<pi_platform>;
+static sycl::detail::SpinLock *PiPlatformsCacheMutex =
+    new sycl::detail::SpinLock;
+static bool PiPlatformCachePopulated = false;
+
 // TODO:: In the following 4 methods we may want to distinguish read access vs.
 // write (as it is OK for multiple threads to read the map without locking it).
 
@@ -821,16 +833,8 @@ pi_result piPlatformsGet(pi_uint32 NumEntries, pi_platform *Platforms,
   // 1. sycl::platform equality issue; we always return the same pi_platform.
   // 2. performance; we can save time by immediately return from cache.
   //
-  // Note: The memory for "PiPlatformsCache" and "PiPlatformsCacheMutex" is
-  // intentionally leaked because the application may call into the SYCL
-  // runtime from a global destructor, and such a call could eventually
-  // access these variables. Therefore, there is no safe time when
-  // "PiPlatformsCache" and "PiPlatformsCacheMutex" could be deleted.
-  static auto PiPlatformsCache = new std::vector<pi_platform>;
-  static auto PiPlatformsCacheMutex = new std::mutex;
-  static bool PiPlatformCachePopulated = false;
-
-  std::lock_guard<std::mutex> Lock(*PiPlatformsCacheMutex);
+
+  const std::lock_guard<sycl::detail::SpinLock> Lock{*PiPlatformsCacheMutex};
   if (!PiPlatformCachePopulated) {
     const char *CommandListCacheSize =
         std::getenv("SYCL_PI_LEVEL_ZERO_MAX_COMMAND_LIST_CACHE");
@@ -5349,4 +5353,18 @@ pi_result piPluginInit(pi_plugin *PluginInit) {
   return PI_SUCCESS;
 }
 
+// SYCL RT calls this api to notify the end of plugin lifetime.
+// It can include all the jobs to tear down resources before
+// the plugin is unloaded from memory.
+pi_result piTearDown(void *PluginParameter) {
+  // reclaim pi_platform objects here since we don't have piPlatformRelease.
+  for (pi_platform &Platform : *PiPlatformsCache) {
+    delete Platform;
+  }
+  delete PiPlatformsCache;
+  delete PiPlatformsCacheMutex;
+
+  return PI_SUCCESS;
+}
+
 } // extern "C"

sycl/plugins/level_zero/pi_level_zero.cpp

@@ -1170,6 +1170,11 @@ pi_result piextProgramGetNativeHandle(pi_program program,
   return piextGetNativeHandle(program, nativeHandle);
 }
 
+// This API is called by Sycl RT to notify the end of the plugin lifetime.
+// TODO: add a global variable lifetime management code here (see
+// pi_level_zero.cpp for reference) Currently this is just a NOOP.
+pi_result piTearDown(void *PluginParameter) { return PI_SUCCESS; }
+
 pi_result piPluginInit(pi_plugin *PluginInit) {
   int CompareVersions = strcmp(PluginInit->PiVersion, SupportedVersion);
   if (CompareVersions < 0) {
@@ -1297,6 +1302,7 @@ pi_result piPluginInit(pi_plugin *PluginInit) {
 
   _PI_CL(piextKernelSetArgMemObj, piextKernelSetArgMemObj)
   _PI_CL(piextKernelSetArgSampler, piextKernelSetArgSampler)
+  _PI_CL(piTearDown, piTearDown)
 
 #undef _PI_CL
 

sycl/plugins/opencl/pi_opencl.cpp

@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include <CL/sycl/detail/device_filter.hpp>
+#include <CL/sycl/detail/pi.hpp>
 #include <CL/sycl/detail/spinlock.hpp>
 #include <detail/global_handler.hpp>
 #include <detail/platform_impl.hpp>
@@ -113,7 +114,18 @@ GlobalHandler::getDeviceFilterList(const std::string &InitValue) {
   return *MDeviceFilterList;
 }
 
-void shutdown() { delete &GlobalHandler::instance(); }
+void shutdown() {
+  for (plugin &Plugin : GlobalHandler::instance().getPlugins()) {
+    // PluginParameter is reserved for future use that can control
+    // some parameters in the plugin tear-down process.
+    // Currently, it is not used.
+    void *PluginParameter = nullptr;
+    Plugin.call_nocheck<PiApiKind::piTearDown>(PluginParameter);
+    Plugin.unload();
+  }
+
+  delete &GlobalHandler::instance();
+}
 
 #ifdef _WIN32
 BOOL WINAPI DllMain(HINSTANCE hinstDLL, DWORD fdwReason, LPVOID lpReserved) {