Support loading llvmlite from the main executable.

llvmlite/binding/__init__.py

@@ -13,4 +13,10 @@
 from .value import *
 from .analysis import *
 from .object_file import *
-from .context import *
+from .context import *
+
+
+def __getattr__(name):
+    if name == 'llvm_version_info':
+        return initfini._version_info()
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

llvmlite/binding/ffi.py

@@ -1,6 +1,8 @@
+import contextlib
 import ctypes
-import threading
 import importlib.resources
+import os
+import threading
 
 from llvmlite.binding.common import _decode_string, _is_shutting_down
 from llvmlite.utils import get_library_name
@@ -77,28 +79,96 @@ def __exit__(self, *exc_details):
         self._lock.release()
 
 
+class _suppress_cleanup_errors:
+    def __init__(self, context):
+        self._context = context
+
+    def __enter__(self):
+        return self._context.__enter__()
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        try:
+            return self._context.__exit__(exc_type, exc_value, traceback)
+        except PermissionError:
+            pass  # Resource dylibs can't be deleted on Windows.
+
+
 class _lib_wrapper(object):
     """Wrap libllvmlite with a lock such that only one thread may access it at
     a time.
 
     This class duck-types a CDLL.
     """
-    __slots__ = ['_lib', '_fntab', '_lock']
 
-    def __init__(self, lib):
-        self._lib = lib
-        self._fntab = {}
+    def __init__(self):
         self._lock = _LLVMLock()
+        self._lib_handle = None
+
+    def _load_lib(self):
+        _lib_load_errors = []
+        # llvmlite native code may exist in one of two places, with the first
+        # taking priority:
+        # 1) In a shared library available as a resource of the llvmlite
+        #    package. This may involve unpacking the shared library from an
+        #    archive.
+        # 2) Linked directly into the main binary. Symbols may be resolved
+        #    from the main binary by passing None as the argument to
+        #    ctypes.CDLL.
+        for lib_context in (
+                _suppress_cleanup_errors(importlib.resources.path(
+                    __name__.rpartition(".")[0], get_library_name())),
+                contextlib.nullcontext(None)):
+            try:
+                with lib_context as lib_path:
+                    if os.name == 'nt' and not lib_path:
+                        # The Windows implementation of ctypes.CDLL does not
+                        # support None as an argument.
+                        continue
+                    self._lib_handle = ctypes.CDLL(lib_path and str(lib_path))
+                    # Check that we can look up expected symbols.
+                    _ = self._lib_handle.LLVMPY_GetVersionInfo()
+                    break
+            except (OSError, AttributeError) as e:
+                # OSError may be raised if the file cannot be opened, or is not
+                # a shared library.
+                # AttributeError is raised if LLVMPY_GetVersionInfo does not
+                # exist.
+                _lib_load_errors.append(e)
+        else:
+            # None of the expected locations contains a loadable version of
+            # llvmlite. Raise OSError with the first error seen during the load
+            # event as the cause of this exception.
+            raise OSError("Could not find/load shared object file") \
+                from _lib_load_errors[0]
+
+    @property
+    def _lib(self):
+        # Not threadsafe.
+        if not self._lib_handle:
+            self._load_lib()
+        return self._lib_handle
+
+    def _resolve(self, lazy_wrapper):
+        """Resolve a lazy wrapper, and store it in the symbol table."""
+        with self._lock:
+            wrapper = getattr(self, lazy_wrapper.symbol_name)
+            if wrapper is lazy_wrapper:
+                cfn = getattr(self._lib, lazy_wrapper.symbol_name)
+                if hasattr(lazy_wrapper, 'argtypes'):
+                    cfn.argtypes = lazy_wrapper.argtypes
+                if hasattr(lazy_wrapper, 'restype'):
+                    cfn.restype = lazy_wrapper.restype
+                if getattr(lazy_wrapper, 'threadsafe', False):
+                    wrapper = cfn
+                else:
+                    wrapper = _lib_fn_wrapper(self._lock, cfn)
+                setattr(self, lazy_wrapper.symbol_name, wrapper)
+            return wrapper
 
     def __getattr__(self, name):
-        try:
-            return self._fntab[name]
-        except KeyError:
-            # Lazily wraps new functions as they are requested
-            cfn = getattr(self._lib, name)
-            wrapped = _lib_fn_wrapper(self._lock, cfn)
-            self._fntab[name] = wrapped
-            return wrapped
+        wrapper = _lazy_lib_fn_wrapper(name)
+        setattr(self, name, wrapper)
+        return wrapper
 
     @property
     def _name(self):
@@ -117,6 +187,16 @@ def _handle(self):
         return self._lib._handle
 
 
+class _lazy_lib_fn_wrapper(object):
+    """A lazy wrapper for a ctypes.CFUNCTYPE that resolves on call."""
+
+    def __init__(self, symbol_name):
+        self.symbol_name = symbol_name
+
+    def __call__(self, *args, **kwargs):
+        return lib._resolve(self)(*args, **kwargs)
+
+
 class _lib_fn_wrapper(object):
     """Wraps and duck-types a ctypes.CFUNCTYPE to provide
     automatic locking when the wrapped function is called.
@@ -151,23 +231,7 @@ def __call__(self, *args, **kwargs):
             return self._cfn(*args, **kwargs)
 
 
-_lib_name = get_library_name()
-
-
-pkgname = ".".join(__name__.split(".")[0:-1])
-try:
-    _lib_handle = importlib.resources.path(pkgname, _lib_name)
-    lib = ctypes.CDLL(str(_lib_handle.__enter__()))
-    # on windows file handles to the dll file remain open after
-    # loading, therefore we can not exit the context manager
-    # which might delete the file
-except OSError as e:
-    msg = f"""Could not find/load shared object file: {_lib_name}
- Error was: {e}"""
-    raise OSError(msg)
-
-
-lib = _lib_wrapper(lib)
+lib = _lib_wrapper()
 
 
 def register_lock_callback(acq_fn, rel_fn):

llvmlite/binding/initfini.py

@@ -68,6 +68,3 @@ def _version_info():
         v.append(x & 0xff)
         x >>= 8
     return tuple(reversed(v))
-
-
-llvm_version_info = _version_info()

llvmlite/tests/__init__.py

@@ -1,5 +1,6 @@
 import sys
 
+import multiprocessing
 import unittest
 from unittest import TestCase
 
@@ -53,5 +54,6 @@ def run_tests(suite=None, xmloutput=None, verbosity=1):
 
 
 def main():
+    multiprocessing.set_start_method('spawn')
     res = run_tests()
     sys.exit(0 if res.wasSuccessful() else 1)

llvmlite/tests/test_loading.py

@@ -0,0 +1,61 @@
+import importlib
+import multiprocessing
+import unittest
+import unittest.mock
+
+from llvmlite import binding as llvm
+
+
+def _test_dylib_resource_loading(result):
+    try:
+        # We must not have loaded the llvmlite dylib yet.
+        assert llvm.ffi.lib._lib_handle is None
+        spec = importlib.util.find_spec(llvm.ffi.__name__.rpartition(".")[0])
+
+        true_dylib = spec.loader.get_resource_reader() \
+            .open_resource(llvm.ffi.get_library_name())
+
+        # A mock resource loader that does not support resource paths
+        class MockResourceReader(importlib.abc.ResourceReader):
+            def is_resource(self, name):
+                return True
+
+            def resource_path(self, name):
+                raise FileNotFoundError
+
+            def open_resource(self, name):
+                return true_dylib
+
+            def contents(self):
+                return []
+
+        # Mock resource loader to force the dylib to be extracted into a
+        # temporary file.
+        with unittest.mock.patch.object(
+                spec.loader, 'get_resource_reader',
+                return_value=MockResourceReader()), \
+             unittest.mock.patch(
+                 'llvmlite.binding.ffi.get_library_name',
+                 return_value='notllvmlite.so'):
+            llvm.llvm_version_info  # force library loading to occur.
+    except Exception as e:
+        result.put(e)
+        raise
+    result.put(None)
+
+
+class TestModuleLoading(unittest.TestCase):
+    def test_dylib_resource_loading(self):
+        subproc_result = multiprocessing.Queue()
+        subproc = multiprocessing.Process(
+            target=_test_dylib_resource_loading,
+            args=(subproc_result,))
+        subproc.start()
+        result = subproc_result.get()
+        subproc.join()
+        if subproc.exitcode:
+            raise result
+
+
+if __name__ == "__main__":
+    unittest.main()