Fixed state pointer pass and added proxy Cython module

lupa/_lupa.pyx

@@ -21,6 +21,9 @@ from cpython.method cimport (
     PyMethod_Check, PyMethod_GET_SELF, PyMethod_GET_FUNCTION)
 from cpython.bytes cimport PyBytes_FromFormat
 
+cdef extern from "Python.h":
+    void *PyLong_AsVoidPtr(object)
+
 #from libc.stdint cimport uintptr_t
 cdef extern from *:
     """
@@ -249,11 +252,14 @@ cdef class LuaRuntime:
                   bint register_eval=True, bint unpack_returned_tuples=False,
                   bint register_builtins=True, overflow_handler=None):
 
-        cdef lua_State *L = <lua_State*>state;
-        self._lua_allocated = False
+        cdef lua_State *L
+
         if state is None:
             self._lua_allocated = True
             L = lua.luaL_newstate()
+        else:
+            self._lua_allocated = False
+            L = <lua_State *> PyLong_AsVoidPtr(state)
 
         if L is NULL:
             raise LuaError("Failed to initialise Lua runtime")

lupa/embedded.pyx

@@ -0,0 +1,88 @@
+import os
+import sys
+from pathlib import Path
+
+
+cdef extern from "Python.h":
+    object PyLong_FromVoidPtr(void *p)
+
+
+runtime = None
+
+
+def _lua_eval_intern(code, *args):
+    # In this scope LuaRuntime returns tuple of (<module_name>, <dll_path>, <return_val>)
+    return runtime.eval(code, *args)[2]
+
+
+cdef public int initialize_lua_runtime(void* L):
+    global runtime
+
+    # Convert pointer to Python object to make possible pass it into LuaRuntime constructor
+    state = PyLong_FromVoidPtr(L)
+
+    print(f"Initialize LuaRuntime at proxy module with lua_State *L = {hex(state)}")
+
+    from lupa import LuaRuntime
+
+    # TODO: Make possible to configure others LuaRuntime options
+    runtime = LuaRuntime(state=state, encoding="latin-1")
+
+
+cdef void setup_system_path():
+    # Add all lua interpreter 'require' paths to Python import paths
+    paths: list[Path] = [Path(it) for it in _lua_eval_intern("package.path").split(";")]
+    for path in set(it.parent for it in paths if it.parent.is_dir()):
+        str_path = str(path)
+        print(f"Append system path: '{str_path}'")
+        sys.path.append(str_path)
+
+
+virtualenv_env_variable = "LUA_PYTHON_VIRTUAL_ENV"
+
+
+cdef void initialize_virtualenv():
+    virtualenv_path = os.environ.get(virtualenv_env_variable)
+    if virtualenv_path is None:
+        print(f"Environment variable '{virtualenv_env_variable}' not set, try to use system Python")
+        return
+
+    this_file = os.path.join(virtualenv_path, "Scripts", "activate_this.py")
+    if not os.path.isfile(this_file):
+        print(f"virtualenv at '{virtualenv_path}' seems corrupted, activation file '{this_file}' not found")
+        return
+
+    print(f"Activate virtualenv at {virtualenv_env_variable}='{virtualenv_path}'")
+    exec(open(this_file).read(), {'__file__': this_file})
+
+
+cdef public int embedded_initialize(void *L):
+    initialize_virtualenv()
+    initialize_lua_runtime(L)
+    setup_system_path()
+    return 1
+
+
+cdef extern from *:
+    """
+    PyMODINIT_FUNC PyInit_embedded(void);
+
+    // use void* to make possible not to link proxy module with lua libraries
+    #define LUA_ENTRY_POINT(x) __declspec(dllexport) int luaopen_ ## x (void *L)
+
+    LUA_ENTRY_POINT(libpylua) {
+        PyImport_AppendInittab("embedded", PyInit_embedded);
+        Py_Initialize();
+        PyImport_ImportModule("embedded");
+        return embedded_initialize(L);
+    }
+    """
+
+# Export proxy DLL name from this .pyd file
+# This name may be used by external Python script installer, e.g.
+#
+# from lupa import embedded
+# dylib_ext = get_dylib_ext_by_os()
+# dest_path = os.path.join(dest_dir, f"{embedded.lua_dylib_name}.{dylib_ext}")
+# shutil.copyfile(embedded.__file__, dest_path)
+lua_dylib_name = "libpylua"

setup.py

@@ -54,6 +54,22 @@ def try_int(s):
         return s
 
 
+def get_option(name):
+    for i, arg in enumerate(sys.argv[1:-1], 1):
+        if arg == name:
+            sys.argv.pop(i)
+            return sys.argv.pop(i)
+    return ""
+
+
+def has_option(name):
+    if name in sys.argv[1:]:
+        sys.argv.remove(name)
+        return True
+    envvar_name = 'LUPA_' + name.lstrip('-').upper().replace('-', '_')
+    return os.environ.get(envvar_name) == 'true'
+
+
 def cmd_output(command):
     """
     Returns the exit code and output of the program, as a triplet of the form
@@ -130,27 +146,32 @@ def lua_libs(package='luajit'):
     return libs_out.split()
 
 
+option_lua_lib = get_option('--lua-lib')
+option_lua_includes = get_option('--lua-includes')
+
+
 def get_lua_build_from_arguments():
-    lua_lib = get_option('--lua-lib')
-    lua_includes = get_option('--lua-includes')
 
-    if not lua_lib or not lua_includes:
+    if not option_lua_lib or not option_lua_includes:
         return []
 
-    print('Using Lua library: %s' % lua_lib)
-    print('Using Lua include directory: %s' % lua_includes)
+    print('Using Lua library: %s' % option_lua_lib)
+    print('Using Lua include directory: %s' % option_lua_includes)
 
-    root, ext = os.path.splitext(lua_lib)
+    root, ext = os.path.splitext(option_lua_lib)
+    libname = os.path.basename(root)
     if os.name == 'nt' and ext == '.lib':
         return [
-            dict(extra_objects=[lua_lib],
-                 include_dirs=[lua_includes],
-                 libfile=lua_lib)
+            dict(extra_objects=[option_lua_lib],
+                 include_dirs=[option_lua_includes],
+                 libfile=option_lua_lib,
+                 libversion=libname)
         ]
     else:
         return [
-            dict(extra_objects=[lua_lib],
-                 include_dirs=[lua_includes])
+            dict(extra_objects=[option_lua_lib],
+                 include_dirs=[option_lua_includes],
+                 libversion=libname)
         ]
 
 
@@ -309,22 +330,6 @@ def use_bundled_lua(path, macros):
     }
 
 
-def get_option(name):
-    for i, arg in enumerate(sys.argv[1:-1], 1):
-        if arg == name:
-            sys.argv.pop(i)
-            return sys.argv.pop(i)
-    return ""
-
-
-def has_option(name):
-    if name in sys.argv[1:]:
-        sys.argv.remove(name)
-        return True
-    envvar_name = 'LUPA_' + name.lstrip('-').upper().replace('-', '_')
-    return os.environ.get(envvar_name) == 'true'
-
-
 c_defines = [
     ('CYTHON_CLINE_IN_TRACEBACK', 0),
 ]
@@ -361,6 +366,29 @@ def has_option(name):
     configs = no_lua_error()
 
 
+try:
+    import Cython.Compiler.Version
+    import Cython.Compiler.Errors as CythonErrors
+    from Cython.Build import cythonize
+    print(f"building with Cython {Cython.Compiler.Version.version}")
+    CythonErrors.LEVEL = 0
+except ImportError:
+    cythonize = None
+    print("ERROR: Can't import cython ... Cython not installed")
+
+
+def do_cythonize(modules: list[Extension], use_cython: bool = True) -> list[Extension]:
+    if not use_cython:
+        print("building without Cython")
+        return modules
+
+    if cythonize is None:
+        print("WARNING: trying to build with Cython, but it is not installed")
+        return modules
+
+    return cythonize(modules)
+
+
 # check if Cython is installed, and use it if requested or necessary
 def prepare_extensions(use_cython=True):
     ext_modules = []
@@ -388,21 +416,7 @@ def prepare_extensions(use_cython=True):
                 print("generated sources not available, need Cython to build")
                 use_cython = True
 
-    cythonize = None
-    if use_cython:
-        try:
-            import Cython.Compiler.Version
-            import Cython.Compiler.Errors as CythonErrors
-            from Cython.Build import cythonize
-            print("building with Cython " + Cython.Compiler.Version.version)
-            CythonErrors.LEVEL = 0
-        except ImportError:
-            print("WARNING: trying to build with Cython, but it is not installed")
-    else:
-        print("building without Cython")
-
-    if cythonize is not None:
-        ext_modules = cythonize(ext_modules)
+        ext_modules = do_cythonize(ext_modules, use_cython)
 
     return ext_modules, ext_libraries
 
@@ -437,6 +451,10 @@ def write_file(filename, content):
 if dll_files:
     extra_setup_args['package_data'] = {'lupa': dll_files}
 
+# Add proxy embedded module to make portal from lua into Python
+embedded_pyx_path = os.path.join("lupa", "embedded.pyx")
+ext_modules += do_cythonize([Extension("lupa.embedded", sources=[embedded_pyx_path])])
+
 # call distutils
 
 setup(