From 71e7fac991d8437e69edeafd9eb183358a09c457 Mon Sep 17 00:00:00 2001
From: Artur Gainullin <artur.gainullin@intel.com>
Date: Fri, 3 Apr 2020 16:42:36 -0700
Subject: [PATCH 1/6] [SYCL] Support connection with multiple plugins

This commit enables including multiple devices of the same device_type
and changed the logic of device selection to just prefer a SYCL_BE
device if present. If someone uses SYCL_BE but appropriate device is
not present, we will simply use any other device.

Signed-off-by: Artur Gainullin <artur.gainullin@intel.com>
---
 sycl/include/CL/sycl/detail/pi.hpp            |  17 +-
 sycl/source/detail/pi.cpp                     | 159 ++++++++++++------
 sycl/source/detail/plugin.hpp                 |  14 +-
 .../program_manager/program_manager.cpp       |   3 +-
 sycl/source/detail/scheduler/commands.cpp     |   2 +-
 sycl/source/device_selector.cpp               | 123 ++++++++++----
 sycl/test/devicelib/assert.cpp                |   4 +-
 sycl/test/regression/image_access.cpp         |   4 +-
 sycl/test/scheduler/HostAccDestruction.cpp    |   2 +-
 sycl/test/scheduler/MemObjRemapping.cpp       |   2 +-
 sycl/test/scheduler/ReleaseResourcesTest.cpp  |   6 +-
 11 files changed, 231 insertions(+), 105 deletions(-)

diff --git a/sycl/include/CL/sycl/detail/pi.hpp b/sycl/include/CL/sycl/detail/pi.hpp
index de77c7671f2b7..444beec6516b5 100644
--- a/sycl/include/CL/sycl/detail/pi.hpp
+++ b/sycl/include/CL/sycl/detail/pi.hpp
@@ -43,6 +43,19 @@ enum class PiApiKind {
 class plugin;
 namespace pi {
 
+// The SYCL_PI_TRACE sets what we will trace.
+// This is a bit-mask of various things we'd want to trace.
+enum TraceLevel {
+  PI_TRACE_BASIC = 0x1,
+  PI_TRACE_CALLS = 0x2,
+  PI_TRACE_ALL = -1
+};
+
+// Return true if we want to trace PI related activities.
+bool trace(TraceLevel level);
+
+const char *traceLabel();
+
 #ifdef SYCL_RT_OS_WINDOWS
 #define OPENCL_PLUGIN_NAME "pi_opencl.dll"
 #define CUDA_PLUGIN_NAME "pi_cuda.dll"
@@ -115,8 +128,8 @@ void *getOsLibraryFuncAddress(void *Library, const std::string &FunctionName);
 // environment variable.
 enum Backend { SYCL_BE_PI_OPENCL, SYCL_BE_PI_CUDA, SYCL_BE_PI_OTHER };
 
-// Check for manually selected BE at run-time.
-bool useBackend(Backend Backend);
+// Get the preferred BE (selected with SYCL_BE).
+Backend getPreferredBE();
 
 // Get a string representing a _pi_platform_info enum
 std::string platformInfoToString(pi_platform_info info);
diff --git a/sycl/source/detail/pi.cpp b/sycl/source/detail/pi.cpp
index 4ec13c447a1c7..01d13d9e47826 100644
--- a/sycl/source/detail/pi.cpp
+++ b/sycl/source/detail/pi.cpp
@@ -22,9 +22,9 @@
 #include <cstring>
 #include <iostream>
 #include <map>
+#include <sstream>
 #include <stddef.h>
 #include <string>
-#include <sstream>
 
 #ifdef XPTI_ENABLE_INSTRUMENTATION
 // Include the headers necessary for emitting
@@ -141,39 +141,80 @@ std::string memFlagsToString(pi_mem_flags Flags) {
   return Sstream.str();
 }
 
-// Check for manually selected BE at run-time.
-static Backend getBackend() {
-  static const char *GetEnv = std::getenv("SYCL_BE");
-  // Current default backend as SYCL_BE_PI_OPENCL
-  // Valid values of GetEnv are "PI_OPENCL", "PI_CUDA" and "PI_OTHER"
-  std::string StringGetEnv = (GetEnv ? GetEnv : "PI_OPENCL");
-  static const Backend Use =
-    std::map<std::string, Backend>{
-      { "PI_OPENCL", SYCL_BE_PI_OPENCL },
-      { "PI_CUDA", SYCL_BE_PI_CUDA },
-      { "PI_OTHER",  SYCL_BE_PI_OTHER }
-    }[ GetEnv ? StringGetEnv : "PI_OPENCL"];
-  return Use;
+// A singleton class to aid that PI configuration parameters
+// are processed only once, like reading a string from environment
+// and converting it into a typed object.
+//
+template <typename T, const char *E> class Config {
+  static Config *m_Instance;
+  T m_Data;
+  Config();
+
+public:
+  static T get() {
+    if (!m_Instance) {
+      m_Instance = new Config();
+    }
+    return m_Instance->m_Data;
+  }
+};
+
+template <typename T, const char *E>
+Config<T, E> *Config<T, E>::m_Instance = nullptr;
+
+// Lists valid configuration environment variables.
+static constexpr char SYCL_BE[] = "SYCL_BE";
+static constexpr char SYCL_INTEROP_BE[] = "SYCL_INTEROP_BE";
+static constexpr char SYCL_PI_TRACE[] = "SYCL_PI_TRACE";
+
+// SYCL_PI_TRACE gives the mask of enabled tracing components (0 default)
+template <> Config<int, SYCL_PI_TRACE>::Config() {
+  const char *Env = std::getenv(SYCL_PI_TRACE);
+  m_Data = (Env ? std::atoi(Env) : 0);
+}
+
+static Backend getBE(const char *EnvVar) {
+  const char *BE = std::getenv(EnvVar);
+  const std::map<std::string, Backend> SyclBeMap{
+      {"PI_OTHER", SYCL_BE_PI_OTHER},
+      {"PI_CUDA", SYCL_BE_PI_CUDA},
+      {"PI_OPENCL", SYCL_BE_PI_OPENCL}};
+  if (BE) {
+    auto It = SyclBeMap.find(BE);
+    if (It == SyclBeMap.end())
+      pi::die("Invalid backend. "
+              "Valid values are PI_OPENCL/PI_CUDA");
+    return It->second;
+  }
+  // Default backend
+  return SYCL_BE_PI_OPENCL;
 }
 
-// Check for manually selected BE at run-time.
-bool useBackend(Backend TheBackend) {
-  return TheBackend == getBackend();
+template <> Config<Backend, SYCL_BE>::Config() { m_Data = getBE(SYCL_BE); }
+
+// SYCL_INTEROP_BE is a way to specify the interoperability plugin.
+template <> Config<Backend, SYCL_INTEROP_BE>::Config() {
+  m_Data = getBE(SYCL_INTEROP_BE);
 }
 
+// Helper interface to not expose "pi::Config" outside of pi.cpp
+Backend getPreferredBE() { return Config<Backend, SYCL_BE>::get(); }
+
 // GlobalPlugin is a global Plugin used with Interoperability constructors that
 // use OpenCL objects to construct SYCL class objects.
 std::shared_ptr<plugin> GlobalPlugin;
 
 // Find the plugin at the appropriate location and return the location.
-// TODO: Change the function appropriately when there are multiple plugins.
-bool findPlugins(vector_class<std::string> &PluginNames) {
+bool findPlugins(vector_class<std::pair<std::string, Backend>> &PluginNames) {
   // TODO: Based on final design discussions, change the location where the
   // plugin must be searched; how to identify the plugins etc. Currently the
   // search is done for libpi_opencl.so/pi_opencl.dll file in LD_LIBRARY_PATH
   // env only.
-  PluginNames.push_back(OPENCL_PLUGIN_NAME);
-  PluginNames.push_back(CUDA_PLUGIN_NAME);
+  //
+  PluginNames.push_back(std::make_pair<std::string, Backend>(
+      OPENCL_PLUGIN_NAME, SYCL_BE_PI_OPENCL));
+  PluginNames.push_back(
+      std::make_pair<std::string, Backend>(CUDA_PLUGIN_NAME, SYCL_BE_PI_CUDA));
   return true;
 }
 
@@ -207,52 +248,66 @@ bool bindPlugin(void *Library, PiPlugin *PluginInformation) {
   return true;
 }
 
-// Load the plugin based on SYCL_BE.
-// TODO: Currently only accepting OpenCL and CUDA plugins. Edit it to identify
-// and load other kinds of plugins, do the required changes in the
-// findPlugins, loadPlugin and bindPlugin functions.
-vector_class<plugin> initialize() {
-  vector_class<plugin> Plugins;
+bool trace(TraceLevel Level) {
+  auto TraceLevelMask = Config<int, SYCL_PI_TRACE>::get();
+  return (TraceLevelMask & Level) == Level;
+}
 
-  if (!useBackend(SYCL_BE_PI_OPENCL) && !useBackend(SYCL_BE_PI_CUDA)) {
-    die("Unknown SYCL_BE");
+const char *traceLabel() {
+  auto TraceLevelMask = Config<int, SYCL_PI_TRACE>::get();
+  switch (TraceLevelMask) {
+  case PI_TRACE_BASIC:
+    return "SYCL_PI_TRACE[PI_TRACE_BASIC]: ";
+  case PI_TRACE_CALLS:
+    return "SYCL_PI_TRACE[PI_TRACE_CALLS]: ";
+  case PI_TRACE_ALL:
+    return "SYCL_PI_TRACE[PI_TRACE_ALL]: ";
+  default:
+    assert("Unsupported trace level");
   }
+  return nullptr;
+}
 
-  bool EnableTrace = (std::getenv("SYCL_PI_TRACE") != nullptr);
-
-  vector_class<std::string> PluginNames;
+// Initializes all available Plugins.
+vector_class<plugin> initialize() {
+  vector_class<plugin> Plugins;
+  vector_class<std::pair<std::string, Backend>> PluginNames;
   findPlugins(PluginNames);
 
-  if (PluginNames.empty() && EnableTrace)
-    std::cerr << "No Plugins Found." << std::endl;
+  if (PluginNames.empty() && trace(PI_TRACE_ALL))
+    std::cerr << traceLabel() << "No Plugins Found." << std::endl;
 
-  PiPlugin PluginInformation; // TODO: include.
+  PiPlugin PluginInformation;
   for (unsigned int I = 0; I < PluginNames.size(); I++) {
-    void *Library = loadPlugin(PluginNames[I]);
+    void *Library = loadPlugin(PluginNames[I].first);
 
     if (!Library) {
-      if (EnableTrace) {
-        std::cerr << "Check if plugin is present. Failed to load plugin: "
-                  << PluginNames[I] << std::endl;
+      if (trace(PI_TRACE_ALL)) {
+        std::cerr << traceLabel() << "Check if plugin is present. "
+                  << "Failed to load plugin: " << PluginNames[I].first
+                  << std::endl;
       }
       continue;
     }
 
-    if (!bindPlugin(Library, &PluginInformation) && EnableTrace) {
-      std::cerr << "Failed to bind PI APIs to the plugin: " << PluginNames[I]
-                << std::endl;
-    }
-    if (useBackend(SYCL_BE_PI_OPENCL) &&
-        PluginNames[I].find("opencl") != std::string::npos) {
-      // Use the OpenCL plugin as the GlobalPlugin
-      GlobalPlugin = std::make_shared<plugin>(PluginInformation);
+    if (!bindPlugin(Library, &PluginInformation)) {
+      if (trace(PI_TRACE_ALL)) {
+        std::cerr << traceLabel() << "Failed to bind PI APIs to the plugin: "
+                  << PluginNames[I].first << std::endl;
+      }
+      continue;
     }
-    if (useBackend(SYCL_BE_PI_CUDA) &&
-        PluginNames[I].find("cuda") != std::string::npos) {
-      // Use the CUDA plugin as the GlobalPlugin
-      GlobalPlugin = std::make_shared<plugin>(PluginInformation);
+    // Set the Global Plugin based on SYCL_INTEROP_BE.
+    // Rework this when it will be explicit in the code which BE is used in the
+    // interoperability methods.
+    if (Config<Backend, SYCL_INTEROP_BE>::get() == PluginNames[I].second) {
+      GlobalPlugin =
+          std::make_shared<plugin>(PluginInformation, PluginNames[I].second);
     }
-    Plugins.push_back(plugin(PluginInformation));
+    Plugins.emplace_back(plugin(PluginInformation, PluginNames[I].second));
+    if (trace(TraceLevel::PI_TRACE_BASIC))
+      std::cerr << traceLabel() << "Plugin found and successfully loaded: "
+                << PluginNames[I].first << std::endl;
   }
 
 #ifdef XPTI_ENABLE_INSTRUMENTATION
diff --git a/sycl/source/detail/plugin.hpp b/sycl/source/detail/plugin.hpp
index 14ddf8f9560e2..b62739891cf68 100644
--- a/sycl/source/detail/plugin.hpp
+++ b/sycl/source/detail/plugin.hpp
@@ -23,9 +23,8 @@ class plugin {
 public:
   plugin() = delete;
 
-  plugin(RT::PiPlugin Plugin) : MPlugin(Plugin) {
-    MPiEnableTrace = (std::getenv("SYCL_PI_TRACE") != nullptr);
-  }
+  plugin(RT::PiPlugin Plugin, RT::Backend UseBackend)
+      : MPlugin(Plugin), MBackend(UseBackend) {}
 
   ~plugin() = default;
 
@@ -52,13 +51,13 @@ class plugin {
   template <PiApiKind PiApiOffset, typename... ArgsT>
   RT::PiResult call_nocheck(ArgsT... Args) const {
     RT::PiFuncInfo<PiApiOffset> PiCallInfo;
-    if (MPiEnableTrace) {
+    if (pi::trace(pi::TraceLevel::PI_TRACE_CALLS)) {
       std::string FnName = PiCallInfo.getFuncName();
       std::cout << "---> " << FnName << "(" << std::endl;
       RT::printArgs(Args...);
     }
     RT::PiResult R = PiCallInfo.getFuncPtr(MPlugin)(Args...);
-    if (MPiEnableTrace) {
+    if (pi::trace(pi::TraceLevel::PI_TRACE_CALLS)) {
       std::cout << ") ---> ";
       RT::printArgs(R);
     }
@@ -74,10 +73,11 @@ class plugin {
     checkPiResult(Err);
   }
 
+  RT::Backend getBackend(void) const { return MBackend; }
+
 private:
   RT::PiPlugin MPlugin;
-  bool MPiEnableTrace;
-
+  const RT::Backend MBackend;
 }; // class plugin
 } // namespace detail
 } // namespace sycl
diff --git a/sycl/source/detail/program_manager/program_manager.cpp b/sycl/source/detail/program_manager/program_manager.cpp
index 064638eb8696f..61822938b6ab2 100644
--- a/sycl/source/detail/program_manager/program_manager.cpp
+++ b/sycl/source/detail/program_manager/program_manager.cpp
@@ -270,7 +270,8 @@ static bool isDeviceBinaryTypeSupported(const context &C,
   }
 
   // OpenCL 2.1 and greater require clCreateProgramWithIL
-  if (pi::useBackend(pi::SYCL_BE_PI_OPENCL) &&
+  pi::Backend CBackend = (detail::getSyclObjImpl(C)->getPlugin()).getBackend();
+  if ((CBackend == pi::SYCL_BE_PI_OPENCL) &&
       C.get_platform().get_info<info::platform::version>() >= "2.1")
     return true;
 
diff --git a/sycl/source/detail/scheduler/commands.cpp b/sycl/source/detail/scheduler/commands.cpp
index 5664ca1ad25e0..de9c98dc8b6a9 100644
--- a/sycl/source/detail/scheduler/commands.cpp
+++ b/sycl/source/detail/scheduler/commands.cpp
@@ -1672,7 +1672,7 @@ cl_int ExecCGCommand::enqueueImp() {
         Requirement *Req = (Requirement *)(Arg.MPtr);
         AllocaCommandBase *AllocaCmd = getAllocaForReq(Req);
         RT::PiMem MemArg = (RT::PiMem)AllocaCmd->getMemAllocation();
-        if (RT::useBackend(pi::Backend::SYCL_BE_PI_OPENCL)) {
+        if (Plugin.getBackend() == (pi::Backend::SYCL_BE_PI_OPENCL)) {
           Plugin.call<PiApiKind::piKernelSetArg>(Kernel, Arg.MIndex,
                                                  sizeof(RT::PiMem), &MemArg);
         } else {
diff --git a/sycl/source/device_selector.cpp b/sycl/source/device_selector.cpp
index 6eb1a32d13471..30511b8827240 100644
--- a/sycl/source/device_selector.cpp
+++ b/sycl/source/device_selector.cpp
@@ -10,23 +10,67 @@
 #include <CL/sycl/device_selector.hpp>
 #include <CL/sycl/exception.hpp>
 #include <CL/sycl/stl.hpp>
+#include <detail/device_impl.hpp>
 #include <detail/force_device.hpp>
 // 4.6.1 Device selection class
 
 __SYCL_INLINE_NAMESPACE(cl) {
 namespace sycl {
+
+// Utility function to check if device is of the preferred SYCL_BE.
+static bool isDeviceOfPreferredSyclBe(const device &Device) {
+  if (Device.is_host())
+    return false;
+
+  detail::pi::Backend PreferredBE = detail::pi::getPreferredBE();
+  detail::pi::Backend DeviceBE =
+      detail::getSyclObjImpl(Device)->getPlugin().getBackend();
+  return PreferredBE == DeviceBE ? true : false;
+}
+
 device device_selector::select_device() const {
   vector_class<device> devices = device::get_devices();
   int score = -1;
   const device *res = nullptr;
-  for (const auto &dev : devices)
-    if (score < operator()(dev)) {
+  for (const auto &dev : devices) {
+    int dev_score = (*this)(dev);
+    if (detail::pi::trace(detail::pi::TraceLevel::PI_TRACE_ALL)) {
+      string_class PlatformVersion = dev.get_info<info::device::platform>()
+                                         .get_info<info::platform::version>();
+      string_class DeviceName = dev.get_info<info::device::name>();
+      std::cout << detail::pi::traceLabel()
+                << "select_device(): -> score = " << score << std::endl
+                << detail::pi::traceLabel() << "  platform: " << PlatformVersion
+                << std::endl
+                << detail::pi::traceLabel() << "  device: " << DeviceName
+                << std::endl;
+    }
+
+    // SYCL spec says: "If more than one device receives the high score then
+    // one of those tied devices will be returned, but which of the devices
+    // from the tied set is to be returned is not defined". Here we give a
+    // preference to the device of the preferred BE.
+    //
+    if (score < dev_score ||
+        (score == dev_score && isDeviceOfPreferredSyclBe(dev))) {
       res = &dev;
-      score = operator()(dev);
+      score = dev_score;
     }
+  }
 
-  if (res != nullptr)
+  if (res != nullptr) {
+    if (detail::pi::trace(detail::pi::TraceLevel::PI_TRACE_BASIC)) {
+      string_class PlatformVersion = res->get_info<info::device::platform>()
+                                         .get_info<info::platform::version>();
+      string_class DeviceName = res->get_info<info::device::name>();
+      std::cout << detail::pi::traceLabel() << "Selected device ->" << std::endl
+                << detail::pi::traceLabel() << "  platform: " << PlatformVersion
+                << std::endl
+                << detail::pi::traceLabel() << "  device: " << DeviceName
+                << std::endl;
+    }
     return *res;
+  }
 
   throw cl::sycl::runtime_error("No device of requested type available.",
                                 PI_DEVICE_NOT_FOUND);
@@ -34,57 +78,70 @@ device device_selector::select_device() const {
 
 int default_selector::operator()(const device &dev) const {
 
-  // Take note of the SYCL_BE environment variable when doing default selection
-  const char *SYCL_BE = std::getenv("SYCL_BE");
-  if (SYCL_BE) {
-    std::string backend = (SYCL_BE ? SYCL_BE : "");
-    // Taking the version information from the platform gives us more useful
-    // information than the driver_version of the device.
-    const platform platform = dev.get_info<info::device::platform>();
-    const std::string platformVersion =
-        platform.get_info<info::platform::version>();;
-    // If using PI_CUDA, don't accept a non-CUDA device
-    if (platformVersion.find("CUDA") == std::string::npos &&
-        backend == "PI_CUDA") {
-      return -1;
-    }
-    // If using PI_OPENCL, don't accept a non-OpenCL device
-    if (platformVersion.find("OpenCL") == std::string::npos &&
-        backend == "PI_OPENCL") {
-      return -1;
-    }
-  }
+  int Score = -1;
+
+  // Give preference to device of SYCL BE.
+  if (isDeviceOfPreferredSyclBe(dev))
+    Score = 50;
 
   // override always wins
   if (dev.get_info<info::device::device_type>() == detail::get_forced_type())
-    return 1000;
+    Score += 1000;
 
   if (dev.is_gpu())
-    return 500;
+    Score += 500;
 
   if (dev.is_cpu())
-    return 300;
+    Score += 300;
 
   if (dev.is_host())
-    return 100;
+    Score += 100;
 
-  return -1;
+  return Score;
 }
 
 int gpu_selector::operator()(const device &dev) const {
-  return dev.is_gpu() ? 1000 : -1;
+  int Score = -1;
+  if (dev.is_gpu()) {
+    Score = 1000;
+    // Give preference to device of SYCL BE.
+    if (isDeviceOfPreferredSyclBe(dev))
+      Score += 50;
+  }
+  return Score;
 }
 
 int cpu_selector::operator()(const device &dev) const {
-  return dev.is_cpu() ? 1000 : -1;
+  int Score = -1;
+  if (dev.is_cpu()) {
+    Score = 1000;
+    // Give preference to device of SYCL BE.
+    if (isDeviceOfPreferredSyclBe(dev))
+      Score += 50;
+  }
+  return Score;
 }
 
 int accelerator_selector::operator()(const device &dev) const {
-  return dev.is_accelerator() ? 1000 : -1;
+  int Score = -1;
+  if (dev.is_accelerator()) {
+    Score = 1000;
+    // Give preference to device of SYCL BE.
+    if (isDeviceOfPreferredSyclBe(dev))
+      Score += 50;
+  }
+  return Score;
 }
 
 int host_selector::operator()(const device &dev) const {
-  return dev.is_host() ? 1000 : -1;
+  int Score = -1;
+  if (dev.is_host()) {
+    Score = 1000;
+    // Give preference to device of SYCL BE.
+    if (isDeviceOfPreferredSyclBe(dev))
+      Score += 50;
+  }
+  return Score;
 }
 
 } // namespace sycl
diff --git a/sycl/test/devicelib/assert.cpp b/sycl/test/devicelib/assert.cpp
index 9170d2557e283..3106f7ac4d1e9 100644
--- a/sycl/test/devicelib/assert.cpp
+++ b/sycl/test/devicelib/assert.cpp
@@ -75,11 +75,11 @@
 //
 // Overall this sounds stable enough. What could possibly go wrong?
 //
-// RUN: env SYCL_PI_TRACE=1 SHOULD_CRASH=1 CL_CONFIG_USE_VECTORIZER=False SYCL_DEVICE_TYPE=CPU EXPECTED_SIGNAL=SIGABRT SKIP_IF_NO_EXT=1 %t.out 2>%t.stderr.native >%t.stdout.native
+// RUN: env SYCL_PI_TRACE=2 SHOULD_CRASH=1 CL_CONFIG_USE_VECTORIZER=False SYCL_DEVICE_TYPE=CPU EXPECTED_SIGNAL=SIGABRT SKIP_IF_NO_EXT=1 %t.out 2>%t.stderr.native >%t.stdout.native
 // RUN: FileCheck %s --input-file %t.stdout.native --check-prefixes=CHECK-NATIVE || FileCheck %s --input-file %t.stderr.native --check-prefix CHECK-NOTSUPPORTED
 // RUN: FileCheck %s --input-file %t.stderr.native --check-prefixes=CHECK-MESSAGE || FileCheck %s --input-file %t.stderr.native --check-prefix CHECK-NOTSUPPORTED
 //
-// RUN: env SYCL_PI_TRACE=1 SYCL_DEVICELIB_INHIBIT_NATIVE=cl_intel_devicelib_assert CL_CONFIG_USE_VECTORIZER=False SYCL_DEVICE_TYPE=CPU EXPECTED_SIGNAL=SIGSEGV %t.out >%t.stdout.pi.fallback
+// RUN: env SYCL_PI_TRACE=2 SYCL_DEVICELIB_INHIBIT_NATIVE=cl_intel_devicelib_assert CL_CONFIG_USE_VECTORIZER=False SYCL_DEVICE_TYPE=CPU EXPECTED_SIGNAL=SIGSEGV %t.out >%t.stdout.pi.fallback
 // RUN: env SHOULD_CRASH=1 SYCL_DEVICELIB_INHIBIT_NATIVE=cl_intel_devicelib_assert CL_CONFIG_USE_VECTORIZER=False SYCL_DEVICE_TYPE=CPU EXPECTED_SIGNAL=SIGSEGV %t.out >%t.stdout.msg.fallback
 // RUN: FileCheck %s --input-file %t.stdout.pi.fallback --check-prefixes=CHECK-FALLBACK
 // RUN: FileCheck %s --input-file %t.stdout.msg.fallback --check-prefixes=CHECK-MESSAGE
diff --git a/sycl/test/regression/image_access.cpp b/sycl/test/regression/image_access.cpp
index aa9303559c422..ee9af31c630be 100644
--- a/sycl/test/regression/image_access.cpp
+++ b/sycl/test/regression/image_access.cpp
@@ -1,7 +1,7 @@
 // RUN: %clangxx -fsycl -fsycl-targets=%sycl_triple %s -o %t.out
 // RUN: env SYCL_DEVICE_TYPE=HOST %t.out
-// RUN: env SYCL_PI_TRACE=1 %CPU_RUN_PLACEHOLDER %t.out 2>&1 %CPU_CHECK_PLACEHOLDER
-// RUN: env SYCL_PI_TRACE=1 %GPU_RUN_PLACEHOLDER %t.out 2>&1 %GPU_CHECK_PLACEHOLDER
+// RUN: env SYCL_PI_TRACE=2 %CPU_RUN_PLACEHOLDER %t.out 2>&1 %CPU_CHECK_PLACEHOLDER
+// RUN: env SYCL_PI_TRACE=2 %GPU_RUN_PLACEHOLDER %t.out 2>&1 %GPU_CHECK_PLACEHOLDER
 // TODO: For now PI checks are skipped for ACC device. To decide if it's good.
 // RUN: env %ACC_RUN_PLACEHOLDER %t.out
 
diff --git a/sycl/test/scheduler/HostAccDestruction.cpp b/sycl/test/scheduler/HostAccDestruction.cpp
index e812816ead5ac..fda1c29298c0c 100644
--- a/sycl/test/scheduler/HostAccDestruction.cpp
+++ b/sycl/test/scheduler/HostAccDestruction.cpp
@@ -1,5 +1,5 @@
 // RUN: %clangxx -fsycl -I %sycl_source_dir %s -o %t.out
-// RUN: env SYCL_PI_TRACE=1 %CPU_RUN_PLACEHOLDER %t.out 2>&1 %CPU_CHECK_PLACEHOLDER
+// RUN: env SYCL_PI_TRACE=2 %CPU_RUN_PLACEHOLDER %t.out 2>&1 %CPU_CHECK_PLACEHOLDER
 //==---------------------- HostAccDestruction.cpp --------------------------==//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
diff --git a/sycl/test/scheduler/MemObjRemapping.cpp b/sycl/test/scheduler/MemObjRemapping.cpp
index e0b49f6b94b62..04061f2a77e4c 100644
--- a/sycl/test/scheduler/MemObjRemapping.cpp
+++ b/sycl/test/scheduler/MemObjRemapping.cpp
@@ -1,5 +1,5 @@
 // RUN: %clangxx -fsycl %s -o %t.out
-// RUN: env SYCL_PI_TRACE=1 %CPU_RUN_PLACEHOLDER %t.out 2>&1 %CPU_CHECK_PLACEHOLDER
+// RUN: env SYCL_PI_TRACE=2 %CPU_RUN_PLACEHOLDER %t.out 2>&1 %CPU_CHECK_PLACEHOLDER
 #include <CL/sycl.hpp>
 #include <cassert>
 #include <cstddef>
diff --git a/sycl/test/scheduler/ReleaseResourcesTest.cpp b/sycl/test/scheduler/ReleaseResourcesTest.cpp
index 9fb6525efe982..b7ced5c5e59b8 100644
--- a/sycl/test/scheduler/ReleaseResourcesTest.cpp
+++ b/sycl/test/scheduler/ReleaseResourcesTest.cpp
@@ -1,8 +1,8 @@
 // RUN: %clangxx -fsycl -fsycl-targets=%sycl_triple -I %sycl_source_dir %s -o %t.out
 // RUN: env SYCL_DEVICE_TYPE=HOST %t.out
-// RUN: env SYCL_PI_TRACE=1 %CPU_RUN_PLACEHOLDER %t.out 2>&1 %CPU_CHECK_PLACEHOLDER
-// RUN: env SYCL_PI_TRACE=1 %GPU_RUN_PLACEHOLDER %t.out 2>&1 %GPU_CHECK_PLACEHOLDER
-// RUN: env SYCL_PI_TRACE=1 %ACC_RUN_PLACEHOLDER %t.out 2>&1 %ACC_CHECK_PLACEHOLDER
+// RUN: env SYCL_PI_TRACE=2 %CPU_RUN_PLACEHOLDER %t.out 2>&1 %CPU_CHECK_PLACEHOLDER
+// RUN: env SYCL_PI_TRACE=2 %GPU_RUN_PLACEHOLDER %t.out 2>&1 %GPU_CHECK_PLACEHOLDER
+// RUN: env SYCL_PI_TRACE=2 %ACC_RUN_PLACEHOLDER %t.out 2>&1 %ACC_CHECK_PLACEHOLDER
 
 // TODO: error: expected string not found in input
 // TODO: PI ---> pi::piProgramCreate(Context, Data, DataLen, &Program)

From 4c24517e05fe97948a06376cf4759a5d1ba47251 Mon Sep 17 00:00:00 2001
From: Artur Gainullin <artur.gainullin@intel.com>
Date: Tue, 21 Apr 2020 13:05:34 -0700
Subject: [PATCH 2/6] Address review comments

* Specialize SYCLConfig template class for SYCL_BE and SYCL_PI_TRACE.
* Reuse SYCLConfig instead of introducing new Config class in PI.
* Use recently introduced backend enum instead of pi::Backend enum.
* Print label instead of number during PI_TRACE.
* Introduce helper that returns label depending on level of tracing
* Force SYCL RT to use specified backend when SYCL_BE is set.
If SYCL_BE is not specified then SYCL RT is not forced to use specific
backend. But make opencl backend preferred.
* Update docs with info about SYCL_BE and SYCL_PI_TRACE

Signed-off-by: Artur Gainullin <artur.gainullin@intel.com>
---
 sycl/doc/EnvironmentVariables.md              | 13 ++-
 sycl/include/CL/sycl/detail/pi.hpp            |  8 +-
 sycl/source/detail/config.cpp                 |  2 +-
 sycl/source/detail/config.def                 |  2 +
 sycl/source/detail/config.hpp                 | 82 +++++++++++++----
 sycl/source/detail/pi.cpp                     | 88 ++++---------------
 sycl/source/detail/plugin.hpp                 |  7 +-
 .../program_manager/program_manager.cpp       |  5 +-
 sycl/source/detail/scheduler/commands.cpp     |  3 +-
 sycl/source/device.cpp                        | 10 +++
 sycl/source/device_selector.cpp               | 10 +--
 11 files changed, 122 insertions(+), 108 deletions(-)

diff --git a/sycl/doc/EnvironmentVariables.md b/sycl/doc/EnvironmentVariables.md
index 0a99f8899f517..60aebb7d2a8e1 100644
--- a/sycl/doc/EnvironmentVariables.md
+++ b/sycl/doc/EnvironmentVariables.md
@@ -11,8 +11,8 @@ subject to change. Do not rely on these variables in production code.
 
 | Environment variable | Values | Description |
 | -------------------- | ------ | ----------- |
-| SYCL_PI_TRACE | Any(\*) | Force tracing of PI calls to stderr. |
-| SYCL_BE | PI_OPENCL, PI_CUDA, PI_OTHER | When SYCL RT is built with PI, this controls which plugin is used by the default device selector. Default value is PI_OPENCL. |
+| SYCL_PI_TRACE | Described [below](#sycl_pi_trace-options)  | Enable specified level of tracing for PI. |
+| SYCL_BE | PI_OPENCL, PI_CUDA | When SYCL RT is built with PI, force SYCL to consider only devices of the specified backend during the device selection. |
 | SYCL_DEVICE_TYPE | One of: CPU, GPU, ACC, HOST | Force SYCL to use the specified device type. If unset, default selection rules are applied. If set to any unlisted value, this control has no effect. If the requested device type is not found, a `cl::sycl::runtime_error` exception is thrown. If a non-default device selector is used, a device must satisfy both the selector and this control to be chosen. This control only has effect on devices created with a selector. |
 | SYCL_PROGRAM_COMPILE_OPTIONS | String of valid OpenCL compile options | Override compile options for all programs. |
 | SYCL_PROGRAM_LINK_OPTIONS | String of valid OpenCL link options | Override link options for all programs. |
@@ -39,3 +39,12 @@ SYCL_PRINT_EXECUTION_GRAPH can accept one or more comma separated values from th
 | after_addHostAcc | print graph after addHostAccessor method |
 | always | print graph before and after each of the above methods |
 
+### SYCL_PI_TRACE Options
+
+SYCL_PI_TRACE can accept one of the values from the table below
+
+| Option | Description |
+| ------ | ----------- |
+| 1 | Enable basic tracing |
+| 2 | Enable tracing of the PI calls |
+| -1 | Enable all levelis of tracing |
diff --git a/sycl/include/CL/sycl/detail/pi.hpp b/sycl/include/CL/sycl/detail/pi.hpp
index 444beec6516b5..a579a8d3e61d9 100644
--- a/sycl/include/CL/sycl/detail/pi.hpp
+++ b/sycl/include/CL/sycl/detail/pi.hpp
@@ -13,6 +13,7 @@
 
 #pragma once
 
+#include <CL/sycl/backend_types.hpp>
 #include <CL/sycl/detail/common.hpp>
 #include <CL/sycl/detail/export.hpp>
 #include <CL/sycl/detail/os_util.hpp>
@@ -124,13 +125,6 @@ void *loadOsLibrary(const std::string &Library);
 // library, implementation is OS dependent.
 void *getOsLibraryFuncAddress(void *Library, const std::string &FunctionName);
 
-// For selection of SYCL RT back-end, now manually through the "SYCL_BE"
-// environment variable.
-enum Backend { SYCL_BE_PI_OPENCL, SYCL_BE_PI_CUDA, SYCL_BE_PI_OTHER };
-
-// Get the preferred BE (selected with SYCL_BE).
-Backend getPreferredBE();
-
 // Get a string representing a _pi_platform_info enum
 std::string platformInfoToString(pi_platform_info info);
 
diff --git a/sycl/source/detail/config.cpp b/sycl/source/detail/config.cpp
index d37626c1526c2..8258bae4ed412 100644
--- a/sycl/source/detail/config.cpp
+++ b/sycl/source/detail/config.cpp
@@ -103,7 +103,7 @@ void readConfig() {
 void dumpConfig() {
 #define CONFIG(Name, MaxSize, CompileTimeDef)                                  \
   {                                                                            \
-    const char *Val = SYCLConfig<Name>::get();                                 \
+    const char *Val = SYCLConfigBase<Name>::getRawValue();                     \
     std::cerr << SYCLConfigBase<Name>::MConfigName << " : "                    \
               << (Val ? Val : "unset") << std::endl;                           \
   }
diff --git a/sycl/source/detail/config.def b/sycl/source/detail/config.def
index 2f0af4d345c8d..32d17018d8639 100644
--- a/sycl/source/detail/config.def
+++ b/sycl/source/detail/config.def
@@ -13,3 +13,5 @@
 CONFIG(SYCL_PRINT_EXECUTION_GRAPH, 32, __SYCL_PRINT_EXECUTION_GRAPH)
 CONFIG(SYCL_DISABLE_EXECUTION_GRAPH_CLEANUP, 1, __SYCL_DISABLE_EXECUTION_GRAPH_CLEANUP)
 CONFIG(SYCL_DEVICE_ALLOWLIST, 1024, __SYCL_DEVICE_ALLOWLIST)
+CONFIG(SYCL_BE, 16, __SYCL_BE)
+CONFIG(SYCL_PI_TRACE, 4, __SYCL_PI_TRACE)
diff --git a/sycl/source/detail/config.hpp b/sycl/source/detail/config.hpp
index 90ec14f8582d8..ffba3c35a8269 100644
--- a/sycl/source/detail/config.hpp
+++ b/sycl/source/detail/config.hpp
@@ -8,9 +8,12 @@
 
 #pragma once
 
+#include <CL/sycl/backend_types.hpp>
 #include <CL/sycl/detail/defines.hpp>
+#include <CL/sycl/detail/pi.hpp>
 
 #include <cstdlib>
+#include <map>
 
 __SYCL_INLINE_NAMESPACE(cl) {
 namespace sycl {
@@ -48,6 +51,9 @@ constexpr const char *getStrOrNullptr(const char *Str) {
   return (Str[0] == '_' && Str[1] == '_') ? nullptr : Str;
 }
 
+// Intializes configs from the configuration file
+void readConfig();
+
 template <ConfigID Config> class SYCLConfigBase;
 
 #define CONFIG(Name, MaxSize, CompileTimeDef)                                  \
@@ -65,38 +71,82 @@ template <ConfigID Config> class SYCLConfigBase;
      * beginning of the string, if it starts with double underscore(__) the    \
      * value is not set.*/                                                     \
     static const char *const MCompileTimeDef;                                  \
+                                                                               \
+    static const char *getRawValue() {                                         \
+      if (ConfigFromEnvEnabled)                                                \
+        if (const char *ValStr = getenv(MConfigName))                          \
+          return ValStr;                                                       \
+                                                                               \
+      if (ConfigFromFileEnabled) {                                             \
+        readConfig();                                                          \
+        if (MValueFromFile)                                                    \
+          return MValueFromFile;                                               \
+      }                                                                        \
+                                                                               \
+      if (ConfigFromCompileDefEnabled && MCompileTimeDef)                      \
+        return MCompileTimeDef;                                                \
+                                                                               \
+      return nullptr;                                                          \
+    }                                                                          \
   };
 #include "config.def"
 #undef CONFIG
 
-// Intializes configs from the configuration file
-void readConfig();
-
 template <ConfigID Config> class SYCLConfig {
   using BaseT = SYCLConfigBase<Config>;
 
 public:
   static const char *get() {
-    const char *ValStr = getRawValue();
+    const char *ValStr = BaseT::getRawValue();
     return ValStr;
   }
+};
 
-private:
-  static const char *getRawValue() {
-    if (ConfigFromEnvEnabled)
-      if (const char *ValStr = getenv(BaseT::MConfigName))
-        return ValStr;
+template <> class SYCLConfig<SYCL_BE> {
+  using BaseT = SYCLConfigBase<SYCL_BE>;
 
-    if (ConfigFromFileEnabled) {
-      readConfig();
-      if (BaseT::MValueFromFile)
-        return BaseT::MValueFromFile;
+public:
+  static backend get() {
+    static bool Initialized = false;
+    static backend Backend = backend::opencl;
+
+    // Configuration parameters are processed only once, like reading a string
+    // from environment and converting it into a typed object.
+    if (Initialized)
+      return Backend;
+
+    const char *ValStr = BaseT::getRawValue();
+    const std::map<std::string, backend> SyclBeMap{
+        {"PI_OPENCL", backend::opencl}, {"PI_CUDA", backend::cuda}};
+    if (ValStr) {
+      auto It = SyclBeMap.find(ValStr);
+      if (It == SyclBeMap.end())
+        pi::die("Invalid backend. "
+                "Valid values are PI_OPENCL/PI_CUDA");
+      Backend = It->second;
     }
+    Initialized = true;
+    return Backend;
+  }
+};
 
-    if (ConfigFromCompileDefEnabled && BaseT::MCompileTimeDef)
-      return BaseT::MCompileTimeDef;
+template <> class SYCLConfig<SYCL_PI_TRACE> {
+  using BaseT = SYCLConfigBase<SYCL_PI_TRACE>;
 
-    return nullptr;
+public:
+  static int get() {
+    static bool Initialized = false;
+    static int Level = 0; // No tracing by default
+
+    // Configuration parameters are processed only once, like reading a string
+    // from environment and converting it into a typed object.
+    if (Initialized)
+      return Level;
+
+    const char *ValStr = BaseT::getRawValue();
+    Level = (ValStr ? std::atoi(ValStr) : 0);
+    Initialized = true;
+    return Level;
   }
 };
 
diff --git a/sycl/source/detail/pi.cpp b/sycl/source/detail/pi.cpp
index 01d13d9e47826..9cb88f613f830 100644
--- a/sycl/source/detail/pi.cpp
+++ b/sycl/source/detail/pi.cpp
@@ -15,6 +15,7 @@
 #include <CL/sycl/context.hpp>
 #include <CL/sycl/detail/common.hpp>
 #include <CL/sycl/detail/pi.hpp>
+#include <detail/config.hpp>
 #include <detail/plugin.hpp>
 
 #include <bitset>
@@ -141,80 +142,21 @@ std::string memFlagsToString(pi_mem_flags Flags) {
   return Sstream.str();
 }
 
-// A singleton class to aid that PI configuration parameters
-// are processed only once, like reading a string from environment
-// and converting it into a typed object.
-//
-template <typename T, const char *E> class Config {
-  static Config *m_Instance;
-  T m_Data;
-  Config();
-
-public:
-  static T get() {
-    if (!m_Instance) {
-      m_Instance = new Config();
-    }
-    return m_Instance->m_Data;
-  }
-};
-
-template <typename T, const char *E>
-Config<T, E> *Config<T, E>::m_Instance = nullptr;
-
-// Lists valid configuration environment variables.
-static constexpr char SYCL_BE[] = "SYCL_BE";
-static constexpr char SYCL_INTEROP_BE[] = "SYCL_INTEROP_BE";
-static constexpr char SYCL_PI_TRACE[] = "SYCL_PI_TRACE";
-
-// SYCL_PI_TRACE gives the mask of enabled tracing components (0 default)
-template <> Config<int, SYCL_PI_TRACE>::Config() {
-  const char *Env = std::getenv(SYCL_PI_TRACE);
-  m_Data = (Env ? std::atoi(Env) : 0);
-}
-
-static Backend getBE(const char *EnvVar) {
-  const char *BE = std::getenv(EnvVar);
-  const std::map<std::string, Backend> SyclBeMap{
-      {"PI_OTHER", SYCL_BE_PI_OTHER},
-      {"PI_CUDA", SYCL_BE_PI_CUDA},
-      {"PI_OPENCL", SYCL_BE_PI_OPENCL}};
-  if (BE) {
-    auto It = SyclBeMap.find(BE);
-    if (It == SyclBeMap.end())
-      pi::die("Invalid backend. "
-              "Valid values are PI_OPENCL/PI_CUDA");
-    return It->second;
-  }
-  // Default backend
-  return SYCL_BE_PI_OPENCL;
-}
-
-template <> Config<Backend, SYCL_BE>::Config() { m_Data = getBE(SYCL_BE); }
-
-// SYCL_INTEROP_BE is a way to specify the interoperability plugin.
-template <> Config<Backend, SYCL_INTEROP_BE>::Config() {
-  m_Data = getBE(SYCL_INTEROP_BE);
-}
-
-// Helper interface to not expose "pi::Config" outside of pi.cpp
-Backend getPreferredBE() { return Config<Backend, SYCL_BE>::get(); }
-
 // GlobalPlugin is a global Plugin used with Interoperability constructors that
 // use OpenCL objects to construct SYCL class objects.
 std::shared_ptr<plugin> GlobalPlugin;
 
 // Find the plugin at the appropriate location and return the location.
-bool findPlugins(vector_class<std::pair<std::string, Backend>> &PluginNames) {
+bool findPlugins(vector_class<std::pair<std::string, backend>> &PluginNames) {
   // TODO: Based on final design discussions, change the location where the
   // plugin must be searched; how to identify the plugins etc. Currently the
   // search is done for libpi_opencl.so/pi_opencl.dll file in LD_LIBRARY_PATH
   // env only.
   //
-  PluginNames.push_back(std::make_pair<std::string, Backend>(
-      OPENCL_PLUGIN_NAME, SYCL_BE_PI_OPENCL));
+  PluginNames.push_back(std::make_pair<std::string, backend>(OPENCL_PLUGIN_NAME,
+                                                             backend::opencl));
   PluginNames.push_back(
-      std::make_pair<std::string, Backend>(CUDA_PLUGIN_NAME, SYCL_BE_PI_CUDA));
+      std::make_pair<std::string, backend>(CUDA_PLUGIN_NAME, backend::cuda));
   return true;
 }
 
@@ -249,12 +191,12 @@ bool bindPlugin(void *Library, PiPlugin *PluginInformation) {
 }
 
 bool trace(TraceLevel Level) {
-  auto TraceLevelMask = Config<int, SYCL_PI_TRACE>::get();
+  auto TraceLevelMask = SYCLConfig<SYCL_PI_TRACE>::get();
   return (TraceLevelMask & Level) == Level;
 }
 
 const char *traceLabel() {
-  auto TraceLevelMask = Config<int, SYCL_PI_TRACE>::get();
+  int TraceLevelMask = SYCLConfig<SYCL_PI_TRACE>::get();
   switch (TraceLevelMask) {
   case PI_TRACE_BASIC:
     return "SYCL_PI_TRACE[PI_TRACE_BASIC]: ";
@@ -271,7 +213,7 @@ const char *traceLabel() {
 // Initializes all available Plugins.
 vector_class<plugin> initialize() {
   vector_class<plugin> Plugins;
-  vector_class<std::pair<std::string, Backend>> PluginNames;
+  vector_class<std::pair<std::string, backend>> PluginNames;
   findPlugins(PluginNames);
 
   if (PluginNames.empty() && trace(PI_TRACE_ALL))
@@ -297,12 +239,16 @@ vector_class<plugin> initialize() {
       }
       continue;
     }
-    // Set the Global Plugin based on SYCL_INTEROP_BE.
-    // Rework this when it will be explicit in the code which BE is used in the
-    // interoperability methods.
-    if (Config<Backend, SYCL_INTEROP_BE>::get() == PluginNames[I].second) {
+    if (SYCLConfig<SYCL_BE>::get() == backend::opencl &&
+        PluginNames[I].first.find("opencl") != std::string::npos) {
+      // Use the OpenCL plugin as the GlobalPlugin
       GlobalPlugin =
-          std::make_shared<plugin>(PluginInformation, PluginNames[I].second);
+          std::make_shared<plugin>(PluginInformation, backend::opencl);
+    }
+    if (SYCLConfig<SYCL_BE>::get() == backend::cuda &&
+        PluginNames[I].first.find("cuda") != std::string::npos) {
+      // Use the CUDA plugin as the GlobalPlugin
+      GlobalPlugin = std::make_shared<plugin>(PluginInformation, backend::cuda);
     }
     Plugins.emplace_back(plugin(PluginInformation, PluginNames[I].second));
     if (trace(TraceLevel::PI_TRACE_BASIC))
diff --git a/sycl/source/detail/plugin.hpp b/sycl/source/detail/plugin.hpp
index b62739891cf68..41b3535eba544 100644
--- a/sycl/source/detail/plugin.hpp
+++ b/sycl/source/detail/plugin.hpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #pragma once
+#include <CL/sycl/backend_types.hpp>
 #include <CL/sycl/detail/common.hpp>
 #include <CL/sycl/detail/pi.hpp>
 #include <CL/sycl/stl.hpp>
@@ -23,7 +24,7 @@ class plugin {
 public:
   plugin() = delete;
 
-  plugin(RT::PiPlugin Plugin, RT::Backend UseBackend)
+  plugin(RT::PiPlugin Plugin, backend UseBackend)
       : MPlugin(Plugin), MBackend(UseBackend) {}
 
   ~plugin() = default;
@@ -73,11 +74,11 @@ class plugin {
     checkPiResult(Err);
   }
 
-  RT::Backend getBackend(void) const { return MBackend; }
+  backend getBackend(void) const { return MBackend; }
 
 private:
   RT::PiPlugin MPlugin;
-  const RT::Backend MBackend;
+  const backend MBackend;
 }; // class plugin
 } // namespace detail
 } // namespace sycl
diff --git a/sycl/source/detail/program_manager/program_manager.cpp b/sycl/source/detail/program_manager/program_manager.cpp
index 61822938b6ab2..29e3f94053940 100644
--- a/sycl/source/detail/program_manager/program_manager.cpp
+++ b/sycl/source/detail/program_manager/program_manager.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include <CL/sycl/backend_types.hpp>
 #include <CL/sycl/context.hpp>
 #include <CL/sycl/detail/common.hpp>
 #include <CL/sycl/detail/os_util.hpp>
@@ -270,8 +271,8 @@ static bool isDeviceBinaryTypeSupported(const context &C,
   }
 
   // OpenCL 2.1 and greater require clCreateProgramWithIL
-  pi::Backend CBackend = (detail::getSyclObjImpl(C)->getPlugin()).getBackend();
-  if ((CBackend == pi::SYCL_BE_PI_OPENCL) &&
+  backend CBackend = (detail::getSyclObjImpl(C)->getPlugin()).getBackend();
+  if ((CBackend == backend::opencl) &&
       C.get_platform().get_info<info::platform::version>() >= "2.1")
     return true;
 
diff --git a/sycl/source/detail/scheduler/commands.cpp b/sycl/source/detail/scheduler/commands.cpp
index de9c98dc8b6a9..0f4534cdc6813 100644
--- a/sycl/source/detail/scheduler/commands.cpp
+++ b/sycl/source/detail/scheduler/commands.cpp
@@ -10,6 +10,7 @@
 
 #include "CL/sycl/access/access.hpp"
 #include <CL/cl.h>
+#include <CL/sycl/backend_types.hpp>
 #include <CL/sycl/detail/kernel_desc.hpp>
 #include <CL/sycl/detail/memory_manager.hpp>
 #include <CL/sycl/detail/stream_impl.hpp>
@@ -1672,7 +1673,7 @@ cl_int ExecCGCommand::enqueueImp() {
         Requirement *Req = (Requirement *)(Arg.MPtr);
         AllocaCommandBase *AllocaCmd = getAllocaForReq(Req);
         RT::PiMem MemArg = (RT::PiMem)AllocaCmd->getMemAllocation();
-        if (Plugin.getBackend() == (pi::Backend::SYCL_BE_PI_OPENCL)) {
+        if (Plugin.getBackend() == backend::opencl) {
           Plugin.call<PiApiKind::piKernelSetArg>(Kernel, Arg.MIndex,
                                                  sizeof(RT::PiMem), &MemArg);
         } else {
diff --git a/sycl/source/device.cpp b/sycl/source/device.cpp
index d08fa5a63922e..da429b814add0 100644
--- a/sycl/source/device.cpp
+++ b/sycl/source/device.cpp
@@ -10,6 +10,7 @@
 #include <CL/sycl/device.hpp>
 #include <CL/sycl/device_selector.hpp>
 #include <CL/sycl/info/info_desc.hpp>
+#include <detail/config.hpp>
 #include <detail/device_impl.hpp>
 #include <detail/force_device.hpp>
 
@@ -50,6 +51,14 @@ vector_class<device> device::get_devices(info::device_type deviceType) {
   if (detail::match_types(deviceType, forced_type)) {
     detail::force_type(deviceType, forced_type);
     for (const auto &plt : platform::get_platforms()) {
+      // If SYCL_BE is set then skip platforms which doesn't have specified
+      // backend.
+      if (std::getenv("SYCL_BE")) {
+        if (plt.is_host() ||
+            detail::getSyclObjImpl(plt)->getPlugin().getBackend() !=
+                detail::SYCLConfig<detail::SYCL_BE>::get())
+          continue;
+      }
       if (includeHost && plt.is_host()) {
         vector_class<device> host_device(
             plt.get_devices(info::device_type::host));
@@ -63,6 +72,7 @@ vector_class<device> device::get_devices(info::device_type deviceType) {
       }
     }
   }
+
   return devices;
 }
 
diff --git a/sycl/source/device_selector.cpp b/sycl/source/device_selector.cpp
index 30511b8827240..dce380f492cf9 100644
--- a/sycl/source/device_selector.cpp
+++ b/sycl/source/device_selector.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include <CL/sycl/backend_types.hpp>
 #include <CL/sycl/device.hpp>
 #include <CL/sycl/device_selector.hpp>
 #include <CL/sycl/exception.hpp>
@@ -17,15 +18,14 @@
 __SYCL_INLINE_NAMESPACE(cl) {
 namespace sycl {
 
-// Utility function to check if device is of the preferred SYCL_BE.
+// Utility function to check if device is of the preferred backend.
+// Currently preference is given to the opencl backend.
 static bool isDeviceOfPreferredSyclBe(const device &Device) {
   if (Device.is_host())
     return false;
 
-  detail::pi::Backend PreferredBE = detail::pi::getPreferredBE();
-  detail::pi::Backend DeviceBE =
-      detail::getSyclObjImpl(Device)->getPlugin().getBackend();
-  return PreferredBE == DeviceBE ? true : false;
+  return detail::getSyclObjImpl(Device)->getPlugin().getBackend() ==
+         backend::opencl;
 }
 
 device device_selector::select_device() const {

From 67d9f2a497572daa15662bf9fa9da6cff1734bae Mon Sep 17 00:00:00 2001
From: Artur Gainullin <artur.gainullin@intel.com>
Date: Tue, 21 Apr 2020 22:30:35 -0700
Subject: [PATCH 3/6] Follow-up fixes

Signed-off-by: Artur Gainullin <artur.gainullin@intel.com>
---
 sycl/doc/EnvironmentVariables.md   |  8 +++----
 sycl/include/CL/sycl/detail/pi.hpp |  2 --
 sycl/source/detail/config.def      |  2 +-
 sycl/source/detail/config.hpp      | 10 +++++---
 sycl/source/detail/pi.cpp          | 37 +++++++++++-------------------
 sycl/source/device.cpp             | 10 ++++----
 sycl/source/device_selector.cpp    | 21 +++++++++--------
 7 files changed, 41 insertions(+), 49 deletions(-)

diff --git a/sycl/doc/EnvironmentVariables.md b/sycl/doc/EnvironmentVariables.md
index 60aebb7d2a8e1..91b2b0e22eb1c 100644
--- a/sycl/doc/EnvironmentVariables.md
+++ b/sycl/doc/EnvironmentVariables.md
@@ -12,7 +12,7 @@ subject to change. Do not rely on these variables in production code.
 | Environment variable | Values | Description |
 | -------------------- | ------ | ----------- |
 | SYCL_PI_TRACE | Described [below](#sycl_pi_trace-options)  | Enable specified level of tracing for PI. |
-| SYCL_BE | PI_OPENCL, PI_CUDA | When SYCL RT is built with PI, force SYCL to consider only devices of the specified backend during the device selection. |
+| SYCL_BE | PI_OPENCL, PI_CUDA | Force SYCL RT to consider only devices of the specified backend during the device selection. |
 | SYCL_DEVICE_TYPE | One of: CPU, GPU, ACC, HOST | Force SYCL to use the specified device type. If unset, default selection rules are applied. If set to any unlisted value, this control has no effect. If the requested device type is not found, a `cl::sycl::runtime_error` exception is thrown. If a non-default device selector is used, a device must satisfy both the selector and this control to be chosen. This control only has effect on devices created with a selector. |
 | SYCL_PROGRAM_COMPILE_OPTIONS | String of valid OpenCL compile options | Override compile options for all programs. |
 | SYCL_PROGRAM_LINK_OPTIONS | String of valid OpenCL link options | Override link options for all programs. |
@@ -41,10 +41,10 @@ SYCL_PRINT_EXECUTION_GRAPH can accept one or more comma separated values from th
 
 ### SYCL_PI_TRACE Options
 
-SYCL_PI_TRACE can accept one of the values from the table below
+SYCL_PI_TRACE accepts a bit-mask. Supported tracing levels are in the table below
 
 | Option | Description |
 | ------ | ----------- |
-| 1 | Enable basic tracing |
+| 1 | Enable basic tracing, which is tracing of PI plugins/devices discovery |
 | 2 | Enable tracing of the PI calls |
-| -1 | Enable all levelis of tracing |
+| -1 | Enable all levels of tracing |
diff --git a/sycl/include/CL/sycl/detail/pi.hpp b/sycl/include/CL/sycl/detail/pi.hpp
index a579a8d3e61d9..55b914d429e3d 100644
--- a/sycl/include/CL/sycl/detail/pi.hpp
+++ b/sycl/include/CL/sycl/detail/pi.hpp
@@ -55,8 +55,6 @@ enum TraceLevel {
 // Return true if we want to trace PI related activities.
 bool trace(TraceLevel level);
 
-const char *traceLabel();
-
 #ifdef SYCL_RT_OS_WINDOWS
 #define OPENCL_PLUGIN_NAME "pi_opencl.dll"
 #define CUDA_PLUGIN_NAME "pi_cuda.dll"
diff --git a/sycl/source/detail/config.def b/sycl/source/detail/config.def
index 32d17018d8639..271eb51fa6530 100644
--- a/sycl/source/detail/config.def
+++ b/sycl/source/detail/config.def
@@ -14,4 +14,4 @@ CONFIG(SYCL_PRINT_EXECUTION_GRAPH, 32, __SYCL_PRINT_EXECUTION_GRAPH)
 CONFIG(SYCL_DISABLE_EXECUTION_GRAPH_CLEANUP, 1, __SYCL_DISABLE_EXECUTION_GRAPH_CLEANUP)
 CONFIG(SYCL_DEVICE_ALLOWLIST, 1024, __SYCL_DEVICE_ALLOWLIST)
 CONFIG(SYCL_BE, 16, __SYCL_BE)
-CONFIG(SYCL_PI_TRACE, 4, __SYCL_PI_TRACE)
+CONFIG(SYCL_PI_TRACE, 16, __SYCL_PI_TRACE)
diff --git a/sycl/source/detail/config.hpp b/sycl/source/detail/config.hpp
index ffba3c35a8269..768c46c2e85cf 100644
--- a/sycl/source/detail/config.hpp
+++ b/sycl/source/detail/config.hpp
@@ -106,14 +106,15 @@ template <> class SYCLConfig<SYCL_BE> {
   using BaseT = SYCLConfigBase<SYCL_BE>;
 
 public:
-  static backend get() {
+  static backend *get() {
     static bool Initialized = false;
+    static bool IsSet = false;
     static backend Backend = backend::opencl;
 
     // Configuration parameters are processed only once, like reading a string
     // from environment and converting it into a typed object.
     if (Initialized)
-      return Backend;
+      return IsSet ? &Backend : nullptr;
 
     const char *ValStr = BaseT::getRawValue();
     const std::map<std::string, backend> SyclBeMap{
@@ -124,9 +125,12 @@ template <> class SYCLConfig<SYCL_BE> {
         pi::die("Invalid backend. "
                 "Valid values are PI_OPENCL/PI_CUDA");
       Backend = It->second;
+      Initialized = true;
+      IsSet = true;
+      return &Backend;
     }
     Initialized = true;
-    return Backend;
+    return nullptr;
   }
 };
 
diff --git a/sycl/source/detail/pi.cpp b/sycl/source/detail/pi.cpp
index 9cb88f613f830..a85590b30e843 100644
--- a/sycl/source/detail/pi.cpp
+++ b/sycl/source/detail/pi.cpp
@@ -195,21 +195,6 @@ bool trace(TraceLevel Level) {
   return (TraceLevelMask & Level) == Level;
 }
 
-const char *traceLabel() {
-  int TraceLevelMask = SYCLConfig<SYCL_PI_TRACE>::get();
-  switch (TraceLevelMask) {
-  case PI_TRACE_BASIC:
-    return "SYCL_PI_TRACE[PI_TRACE_BASIC]: ";
-  case PI_TRACE_CALLS:
-    return "SYCL_PI_TRACE[PI_TRACE_CALLS]: ";
-  case PI_TRACE_ALL:
-    return "SYCL_PI_TRACE[PI_TRACE_ALL]: ";
-  default:
-    assert("Unsupported trace level");
-  }
-  return nullptr;
-}
-
 // Initializes all available Plugins.
 vector_class<plugin> initialize() {
   vector_class<plugin> Plugins;
@@ -217,7 +202,8 @@ vector_class<plugin> initialize() {
   findPlugins(PluginNames);
 
   if (PluginNames.empty() && trace(PI_TRACE_ALL))
-    std::cerr << traceLabel() << "No Plugins Found." << std::endl;
+    std::cerr << "SYCL_PI_TRACE[all]: "
+              << "No Plugins Found." << std::endl;
 
   PiPlugin PluginInformation;
   for (unsigned int I = 0; I < PluginNames.size(); I++) {
@@ -225,7 +211,8 @@ vector_class<plugin> initialize() {
 
     if (!Library) {
       if (trace(PI_TRACE_ALL)) {
-        std::cerr << traceLabel() << "Check if plugin is present. "
+        std::cerr << "SYCL_PI_TRACE[all]: "
+                  << "Check if plugin is present. "
                   << "Failed to load plugin: " << PluginNames[I].first
                   << std::endl;
       }
@@ -234,25 +221,27 @@ vector_class<plugin> initialize() {
 
     if (!bindPlugin(Library, &PluginInformation)) {
       if (trace(PI_TRACE_ALL)) {
-        std::cerr << traceLabel() << "Failed to bind PI APIs to the plugin: "
+        std::cerr << "SYCL_PI_TRACE[all]: "
+                  << "Failed to bind PI APIs to the plugin: "
                   << PluginNames[I].first << std::endl;
       }
       continue;
     }
-    if (SYCLConfig<SYCL_BE>::get() == backend::opencl &&
-        PluginNames[I].first.find("opencl") != std::string::npos) {
+    backend *BE = SYCLConfig<SYCL_BE>::get();
+    if (!BE || (*BE == backend::opencl &&
+                PluginNames[I].first.find("opencl") != std::string::npos)) {
       // Use the OpenCL plugin as the GlobalPlugin
       GlobalPlugin =
           std::make_shared<plugin>(PluginInformation, backend::opencl);
-    }
-    if (SYCLConfig<SYCL_BE>::get() == backend::cuda &&
-        PluginNames[I].first.find("cuda") != std::string::npos) {
+    } else if (*BE == backend::cuda &&
+               PluginNames[I].first.find("cuda") != std::string::npos) {
       // Use the CUDA plugin as the GlobalPlugin
       GlobalPlugin = std::make_shared<plugin>(PluginInformation, backend::cuda);
     }
     Plugins.emplace_back(plugin(PluginInformation, PluginNames[I].second));
     if (trace(TraceLevel::PI_TRACE_BASIC))
-      std::cerr << traceLabel() << "Plugin found and successfully loaded: "
+      std::cerr << "SYCL_PI_TRACE[basic]: "
+                << "Plugin found and successfully loaded: "
                 << PluginNames[I].first << std::endl;
   }
 
diff --git a/sycl/source/device.cpp b/sycl/source/device.cpp
index da429b814add0..91f2cc453bbb0 100644
--- a/sycl/source/device.cpp
+++ b/sycl/source/device.cpp
@@ -53,12 +53,12 @@ vector_class<device> device::get_devices(info::device_type deviceType) {
     for (const auto &plt : platform::get_platforms()) {
       // If SYCL_BE is set then skip platforms which doesn't have specified
       // backend.
-      if (std::getenv("SYCL_BE")) {
-        if (plt.is_host() ||
-            detail::getSyclObjImpl(plt)->getPlugin().getBackend() !=
-                detail::SYCLConfig<detail::SYCL_BE>::get())
+      backend *ForcedBackend = detail::SYCLConfig<detail::SYCL_BE>::get();
+      if (ForcedBackend)
+        if (!plt.is_host() &&
+            (detail::getSyclObjImpl(plt)->getPlugin().getBackend() !=
+             *ForcedBackend))
           continue;
-      }
       if (includeHost && plt.is_host()) {
         vector_class<device> host_device(
             plt.get_devices(info::device_type::host));
diff --git a/sycl/source/device_selector.cpp b/sycl/source/device_selector.cpp
index dce380f492cf9..aba27e0c926fa 100644
--- a/sycl/source/device_selector.cpp
+++ b/sycl/source/device_selector.cpp
@@ -38,12 +38,12 @@ device device_selector::select_device() const {
       string_class PlatformVersion = dev.get_info<info::device::platform>()
                                          .get_info<info::platform::version>();
       string_class DeviceName = dev.get_info<info::device::name>();
-      std::cout << detail::pi::traceLabel()
+      std::cout << "SYCL_PI_TRACE[all]: "
                 << "select_device(): -> score = " << score << std::endl
-                << detail::pi::traceLabel() << "  platform: " << PlatformVersion
-                << std::endl
-                << detail::pi::traceLabel() << "  device: " << DeviceName
-                << std::endl;
+                << "SYCL_PI_TRACE[all]: "
+                << "  platform: " << PlatformVersion << std::endl
+                << "SYCL_PI_TRACE[all]: "
+                << "  device: " << DeviceName << std::endl;
     }
 
     // SYCL spec says: "If more than one device receives the high score then
@@ -63,11 +63,12 @@ device device_selector::select_device() const {
       string_class PlatformVersion = res->get_info<info::device::platform>()
                                          .get_info<info::platform::version>();
       string_class DeviceName = res->get_info<info::device::name>();
-      std::cout << detail::pi::traceLabel() << "Selected device ->" << std::endl
-                << detail::pi::traceLabel() << "  platform: " << PlatformVersion
-                << std::endl
-                << detail::pi::traceLabel() << "  device: " << DeviceName
-                << std::endl;
+      std::cout << "SYCL_PI_TRACE[all]: "
+                << "Selected device ->" << std::endl
+                << "SYCL_PI_TRACE[all]: "
+                << "  platform: " << PlatformVersion << std::endl
+                << "SYCL_PI_TRACE[all]: "
+                << "  device: " << DeviceName << std::endl;
     }
     return *res;
   }

From d27517d7bcd699c0b0b1a15d439ece139554b92d Mon Sep 17 00:00:00 2001
From: Artur Gainullin <artur.gainullin@intel.com>
Date: Wed, 22 Apr 2020 09:32:28 -0700
Subject: [PATCH 4/6] Fix subdevice_pi test

Signed-off-by: Artur Gainullin <artur.gainullin@intel.com>
---
 sycl/test/basic_tests/subdevice_pi.cpp | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/sycl/test/basic_tests/subdevice_pi.cpp b/sycl/test/basic_tests/subdevice_pi.cpp
index af58fcc68a32f..143bc156de194 100644
--- a/sycl/test/basic_tests/subdevice_pi.cpp
+++ b/sycl/test/basic_tests/subdevice_pi.cpp
@@ -1,7 +1,7 @@
 // RUN: %clangxx -fsycl -fsycl-targets=%sycl_triple %s -o %t.out
-// RUN: %CPU_RUN_PLACEHOLDER SYCL_PI_TRACE=1 %t.out separate equally %CPU_CHECK_PLACEHOLDER --check-prefix CHECK-SEPARATE
-// RUN: %CPU_RUN_PLACEHOLDER SYCL_PI_TRACE=1 %t.out shared equally %CPU_CHECK_PLACEHOLDER --check-prefix CHECK-SHARED --implicit-check-not piContextCreate --implicit-check-not piMemBufferCreate
-// RUN: %CPU_RUN_PLACEHOLDER SYCL_PI_TRACE=1 %t.out fused  equally %CPU_CHECK_PLACEHOLDER --check-prefix CHECK-FUSED --implicit-check-not piContextCreate --implicit-check-not piMemBufferCreate
+// RUN: %CPU_RUN_PLACEHOLDER SYCL_PI_TRACE=2 %t.out separate equally %CPU_CHECK_PLACEHOLDER --check-prefix CHECK-SEPARATE
+// RUN: %CPU_RUN_PLACEHOLDER SYCL_PI_TRACE=2 %t.out shared equally %CPU_CHECK_PLACEHOLDER --check-prefix CHECK-SHARED --implicit-check-not piContextCreate --implicit-check-not piMemBufferCreate
+// RUN: %CPU_RUN_PLACEHOLDER SYCL_PI_TRACE=2 %t.out fused  equally %CPU_CHECK_PLACEHOLDER --check-prefix CHECK-FUSED --implicit-check-not piContextCreate --implicit-check-not piMemBufferCreate
 //
 // Intel OpenCL CPU Runtime supports device partition on all (multi-core)
 // platforms. Other devices may not support this.

From cb7b4eb74dcda46006553530ed134e79df44ffd2 Mon Sep 17 00:00:00 2001
From: Artur Gainullin <artur.gainullin@intel.com>
Date: Wed, 22 Apr 2020 13:47:15 -0700
Subject: [PATCH 5/6] Address comments

Signed-off-by: Artur Gainullin <artur.gainullin@intel.com>
---
 sycl/source/detail/config.hpp           | 29 ++++++++++++++-----------
 sycl/test/basic_tests/event_release.cpp |  2 +-
 2 files changed, 17 insertions(+), 14 deletions(-)

diff --git a/sycl/source/detail/config.hpp b/sycl/source/detail/config.hpp
index 768c46c2e85cf..8ee7c943f8d83 100644
--- a/sycl/source/detail/config.hpp
+++ b/sycl/source/detail/config.hpp
@@ -12,8 +12,8 @@
 #include <CL/sycl/detail/defines.hpp>
 #include <CL/sycl/detail/pi.hpp>
 
+#include <algorithm>
 #include <cstdlib>
-#include <map>
 
 __SYCL_INLINE_NAMESPACE(cl) {
 namespace sycl {
@@ -97,7 +97,7 @@ template <ConfigID Config> class SYCLConfig {
 
 public:
   static const char *get() {
-    const char *ValStr = BaseT::getRawValue();
+    static const char *ValStr = BaseT::getRawValue();
     return ValStr;
   }
 };
@@ -108,29 +108,30 @@ template <> class SYCLConfig<SYCL_BE> {
 public:
   static backend *get() {
     static bool Initialized = false;
-    static bool IsSet = false;
-    static backend Backend = backend::opencl;
+    static backend *BackendPtr = nullptr;
 
     // Configuration parameters are processed only once, like reading a string
     // from environment and converting it into a typed object.
     if (Initialized)
-      return IsSet ? &Backend : nullptr;
+      return BackendPtr;
 
     const char *ValStr = BaseT::getRawValue();
-    const std::map<std::string, backend> SyclBeMap{
-        {"PI_OPENCL", backend::opencl}, {"PI_CUDA", backend::cuda}};
+    const std::array<std::pair<std::string, backend>, 2> SyclBeMap = {
+        {{"PI_OPENCL", backend::opencl}, {"PI_CUDA", backend::cuda}}};
     if (ValStr) {
-      auto It = SyclBeMap.find(ValStr);
+      auto It = std::find_if(
+          std::begin(SyclBeMap), std::end(SyclBeMap),
+          [&ValStr](const std::pair<std::string, backend> &element) {
+            return element.first == ValStr;
+          });
       if (It == SyclBeMap.end())
         pi::die("Invalid backend. "
                 "Valid values are PI_OPENCL/PI_CUDA");
-      Backend = It->second;
-      Initialized = true;
-      IsSet = true;
-      return &Backend;
+      static backend Backend = It->second;
+      BackendPtr = &Backend;
     }
     Initialized = true;
-    return nullptr;
+    return BackendPtr;
   }
 };
 
@@ -140,6 +141,8 @@ template <> class SYCLConfig<SYCL_PI_TRACE> {
 public:
   static int get() {
     static bool Initialized = false;
+    // We don't use TraceLevel enum here because user can provide any bitmask
+    // which can correspond to several enum values.
     static int Level = 0; // No tracing by default
 
     // Configuration parameters are processed only once, like reading a string
diff --git a/sycl/test/basic_tests/event_release.cpp b/sycl/test/basic_tests/event_release.cpp
index 37e0d4258b28b..9693516a54353 100644
--- a/sycl/test/basic_tests/event_release.cpp
+++ b/sycl/test/basic_tests/event_release.cpp
@@ -1,5 +1,5 @@
 // RUN: %clangxx -fsycl %s -o %t.out
-// RUN: env SYCL_PI_TRACE=1 %CPU_RUN_PLACEHOLDER %t.out 2>&1 %CPU_CHECK_PLACEHOLDER
+// RUN: env SYCL_PI_TRACE=2 %CPU_RUN_PLACEHOLDER %t.out 2>&1 %CPU_CHECK_PLACEHOLDER
 #include <CL/sycl.hpp>
 #include <cassert>
 #include <iostream>

From 3677f670823f496446a37bcfa233d2cebfd40115 Mon Sep 17 00:00:00 2001
From: Artur Gainullin <artur.gainullin@intel.com>
Date: Wed, 22 Apr 2020 15:14:22 -0700
Subject: [PATCH 6/6] Include missed headers

Signed-off-by: Artur Gainullin <artur.gainullin@intel.com>
---
 sycl/source/detail/config.hpp | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/sycl/source/detail/config.hpp b/sycl/source/detail/config.hpp
index 8ee7c943f8d83..e1d571d8dfb3d 100644
--- a/sycl/source/detail/config.hpp
+++ b/sycl/source/detail/config.hpp
@@ -13,7 +13,9 @@
 #include <CL/sycl/detail/pi.hpp>
 
 #include <algorithm>
+#include <array>
 #include <cstdlib>
+#include <utility>
 
 __SYCL_INLINE_NAMESPACE(cl) {
 namespace sycl {