diff --git a/vta/python/vta/testing/simulator.py b/vta/python/vta/testing/simulator.py
index 7f2471c61532..05f37c182815 100644
--- a/vta/python/vta/testing/simulator.py
+++ b/vta/python/vta/testing/simulator.py
@@ -17,6 +17,7 @@
 """Utilities to start simulator."""
 import ctypes
 import json
+import warnings
 import tvm
 from ..environment import get_env
 from ..libinfo import find_libvta
@@ -27,25 +28,30 @@ def _load_sw():
 
     env = get_env()
     lib_driver_name = "libvta_tsim" if env.TARGET == "tsim" else "libvta_fsim"
+    require_sim = env.TARGET in ("sim", "tsim")
+    libs = []
 
     # Load driver library
-    lib_driver = find_libvta(lib_driver_name, optional=True)
-    assert lib_driver
+    lib_driver = find_libvta(lib_driver_name, optional=(not require_sim))
+
+    if not lib_driver:
+        return []
+
     try:
         libs = [ctypes.CDLL(lib_driver[0], ctypes.RTLD_GLOBAL)]
-    except OSError:
+    except OSError as err:
+        if require_sim:
+            raise err
+        warnings.warn("Error when loading VTA driver {}: {}".format(lib_driver[0], err))
         return []
 
     if env.TARGET == "tsim":
         lib_hw = find_libvta("libvta_hw", optional=True)
         assert lib_hw  # make sure to make in ${VTA_HW_PATH}/hardware/chisel
-        try:
-            f = tvm.get_global_func("vta.tsim.init")
-            m = tvm.runtime.load_module(lib_hw[0], "vta-tsim")
-            f(m)
-            return lib_hw
-        except OSError:
-            return []
+        f = tvm.get_global_func("vta.tsim.init")
+        m = tvm.runtime.load_module(lib_hw[0], "vta-tsim")
+        f(m)
+        return lib_hw
 
     return libs