Initial merge with xeus-clang-repl

Author: alexander-penev

.github/workflows/main.yml

@@ -15,9 +15,9 @@ concurrency:
 
 jobs:
   unix:
-    
+
     runs-on: ${{ matrix.os }}
-     
+
     strategy:
       fail-fast: false
       matrix:
@@ -51,10 +51,10 @@ jobs:
           mkdir -p build
           cd build
           cmake .. \
-           -DCMAKE_PREFIX_PATH=$CONDA_PREFIX \
-           -DCMAKE_INSTALL_PREFIX=$CONDA_PREFIX \
-           -DXEUS_CPP_ENABLE_CODE_COVERAGE=${{ matrix.coverage }} \
-           ${{ matrix.extra_cmake_flags }}
+            -DCMAKE_PREFIX_PATH=$CONDA_PREFIX \
+            -DCMAKE_INSTALL_PREFIX=$CONDA_PREFIX \
+            -DXEUS_CPP_ENABLE_CODE_COVERAGE=${{ matrix.coverage }} \
+            ${{ matrix.extra_cmake_flags }}
 
       - name: build
         shell: bash -l {0}
@@ -72,14 +72,14 @@ jobs:
         shell: bash -l {0}
         run: |
           cd build/test
+          export CPLUS_INCLUDE_PATH=$CONDA_PREFIX/include:$CONDA_BUILD_SYSROOT:$CONDA_BUILD_SYSROOT/..:$CONDA_BUILD_SYSROOT/../x86_64-conda-linux-gnu/include/c++/12.3.0:$CONDA_BUILD_SYSROOT/../x86_64-conda-linux-gnu/include/c++/12.3.0/include:$CONDA_BUILD_SYSROOT/../include:$CONDA_BUILD_SYSROOT/usr/include:/home/runner/micromamba-root/envs/xeus-cpp/bin/../lib/gcc/x86_64-conda-linux-gnu/12.3.0/include:/home/runner/micromamba-root/envs/xeus-cpp/bin/../lib/gcc/x86_64-conda-linux-gnu/12.3.0/include-fixed:/home/runner/micromamba-root/envs/xeus-cpp/bin/../lib/gcc/x86_64-conda-linux-gnu/12.3.0/../../../../x86_64-conda-linux-gnu/include:/home/runner/micromamba-root/envs/xeus-cpp/bin/../x86_64-conda-linux-gnu/sysroot/usr/include
           ./test_xeus_cpp
         timeout-minutes: 4
 
       - name: test
         shell: bash -l {0}
         run: |
-          cd test
-          pytest .  --reruns 5
+          $CONDA_PREFIX/bin/pytest test --reruns 5
 
       - name: Prepare code coverage report
         if: ${{ success() && (matrix.coverage == true) }}
@@ -167,15 +167,15 @@ jobs:
       - name: Setup emsdk
         shell: bash -l {0}
         run: |
-          emsdk install ${{matrix.emsdk_ver}}     
+          emsdk install ${{matrix.emsdk_ver}}
 
       - name: Build xeus-cpp
         shell: bash -l {0}
         run: |
           emsdk activate ${{matrix.emsdk_ver}}
           source $CONDA_EMSDK_DIR/emsdk_env.sh
           micromamba create -f environment-wasm-host.yml --platform=emscripten-wasm32
-          
+
           mkdir build
           pushd build
 
@@ -192,3 +192,9 @@ jobs:
             -DCMAKE_FIND_ROOT_PATH_MODE_PACKAGE=ON            \
             ..
           EMCC_CFLAGS='-sERROR_ON_UNDEFINED_SYMBOLS=0' emmake make -j5
+
+      - name: Setup tmate session
+        if: ${{ failure() && runner.debug }}
+        uses: mxschmitt/action-tmate@v3
+        # When debugging increase to a suitable value!
+        timeout-minutes: 30

CMakeLists.txt

@@ -12,6 +12,12 @@ set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake;${CMAKE_MODULE_PATH}")
 
 set(XEUS_CPP_INCLUDE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/include)
 
+enable_language(CXX)
+set(CMAKE_CXX_EXTENSIONS NO)
+set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+
+include(GNUInstallDirs)
+
 # Versionning
 # ===========
 
@@ -26,29 +32,15 @@ set(${PROJECT_NAME}_VERSION
     ${XEUS_CPP_VERSION_MAJOR}.${XEUS_CPP_VERSION_MINOR}.${XEUS_CPP_VERSION_PATCH})
 message(STATUS "Building xeus-cpp v${${PROJECT_NAME}_VERSION}")
 
-# Configuration
-# =============
-
-include(GNUInstallDirs)
-
-if (NOT DEFINED XEUS_CPP_KERNELSPEC_PATH)
-    set(XEUS_CPP_KERNELSPEC_PATH "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_BINDIR}/")
-endif ()
-
-configure_file (
-    "${CMAKE_CURRENT_SOURCE_DIR}/share/jupyter/kernels/xcpp/kernel.json.in"
-    "${CMAKE_CURRENT_SOURCE_DIR}/share/jupyter/kernels/xcpp/kernel.json"
-)
-
 # Build options
 # =============
 
 option(XEUS_CPP_BUILD_STATIC "Build xeus-cpp static library" ON)
 OPTION(XEUS_CPP_BUILD_SHARED "Split xcpp build into executable and library" ON)
 OPTION(XEUS_CPP_BUILD_EXECUTABLE "Build the xcpp executable" ON)
 
-OPTION(XEUS_CPP_USE_SHARED_XEUS "Link xcpp  with the xeus shared library (instead of the static library)" ON)
-OPTION(XEUS_CPP_USE_SHARED_XEUS_CPP "Link xcpp  with the xeus shared library (instead of the static library)" ON)
+OPTION(XEUS_CPP_USE_SHARED_XEUS "Link xcpp with the xeus shared library (instead of the static library)" ON)
+OPTION(XEUS_CPP_USE_SHARED_XEUS_CPP "Link xcpp with the xeus shared library (instead of the static library)" ON)
 
 # Test options
 OPTION(XEUS_CPP_BUILD_TESTS "xeus-cpp test suite" ON)
@@ -77,7 +69,6 @@ if(EMSCRIPTEN)
     SET(XEUS_CPP_BUILD_TESTS OFF)
 endif()
 
-
 # Dependencies
 # ============
 
@@ -103,15 +94,17 @@ else()
   add_compile_options(-fexceptions)
 endif ()
 
-if (CMAKE_CXX_COMPILER_ID MATCHES "Clang" OR CMAKE_CXX_COMPILER_ID MATCHES "GNU" OR CMAKE_CXX_COMPILER_ID MATCHES "Intel")
+if (CMAKE_CXX_COMPILER_ID MATCHES "Clang" OR
+    CMAKE_CXX_COMPILER_ID MATCHES "GNU" OR
+    CMAKE_CXX_COMPILER_ID MATCHES "Intel")
     if(NOT EMSCRIPTEN)
         add_compile_options(-Wunused-parameter -Wextra -Wreorder)
     endif()
-    
 
     CHECK_CXX_COMPILER_FLAG("-std=c++17" HAS_CPP_17_FLAG)
     if (HAS_CPP_17_FLAG)
-        add_compile_options(-std=c++17)
+        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17")
+        set(CMAKE_CXX_STANDARD 17)
     else ()
         message(FATAL_ERROR "Unsupported compiler -- xeus requires C++17 support!")
     endif ()
@@ -120,21 +113,85 @@ endif ()
 if(EMSCRIPTEN)
     # ENV (https://github.com/emscripten-core/emscripten/commit/6d9681ad04f60b41ef6345ab06c29bbc9eeb84e0)
     set(EMSCRIPTEN_FEATURES "${EMSCRIPTEN_FEATURES} -s \"EXTRA_EXPORTED_RUNTIME_METHODS=[ENV']\"")
+    set(CMAKE_FIND_LIBRARY_SUFFIXES ".a")
+endif()
+
+find_package(CppInterOp REQUIRED CONFIG PATHS "${CMAKE_PREFIX_PATH}" "${CMAKE_PREFIX_PATH}/lib" "${CPPINTEROP_DIR}" "${CPPINTEROP_DIR}/lib")
+if(CppInterOp_FOUND)
+  set(CppInterOp_CLANG_VERSION "17")
+  message(STATUS "Found CppInterOp: config=${CppInterOp_CONFIG} dir=${CppInterOp_DIR} (found version=${CppInterOp_VERSION} compatible with Clang ${CppInterOp_CLANG_VERSION}.x)")
 endif()
 
-find_package(Clang REQUIRED)
-include(AddLLVM)
-include(HandleLLVMOptions)
-add_definitions(${LLVM_DEFINITIONS})
 find_package(argparse REQUIRED)
 find_package(pugixml REQUIRED)
 
+set(Python_FIND_VIRTUALENV ONLY)
+find_package(Python COMPONENTS Interpreter Development)
+
+# Configuration
+# =============
+
+if (NOT DEFINED XEUS_CPP_KERNELSPEC_PATH)
+    set(XEUS_CPP_KERNELSPEC_PATH "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_BINDIR}/")
+    set(XEUS_CPP_RESOURCE_DIR ${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}/clang/${CppInterOp_CLANG_VERSION})
+endif ()
+
+function(configure_kernel kernel)
+  set(XEUS_CPP_CPLUS_INCLUDE_PATH "$ENV{CPLUS_INCLUDE_PATH}")
+  set(XEUS_CPP_PATH "$ENV{PATH}")
+  set(XEUS_CPP_LD_LIBRARY_PATH "$ENV{LD_LIBRARY_PATH}")
+  set(XEUS_CPP_PYTHONPATH "$ENV{PYTHONPATH}")
+  set(XEUS_CPP_VENV_PATH "$ENV{VENV}")
+
+  message(STATUS "Debug: Replace in kernels")
+  message(STATUS "Debug: XEUS_CPP_CPLUS_INCLUDE_PATH=${XEUS_CPP_CPLUS_INCLUDE_PATH}")
+  message(STATUS "Debug: XEUS_CPP_PATH=${XEUS_CPP_PATH}")
+  message(STATUS "Debug: XEUS_CPP_LD_LIBRARY_PATH=${XEUS_CPP_LD_LIBRARY_PATH}")
+  message(STATUS "Debug: XEUS_CPP_PYTHONPATH=${XEUS_CPP_PYTHONPATH}")
+  message(STATUS "Debug: XEUS_CPP_VENV_PATH=${XEUS_CPP_VENV_PATH}")
+  message(STATUS "Debug: CMAKE_CURRENT_SOURCE_DIR=${CMAKE_CURRENT_SOURCE_DIR}")
+  message(STATUS "Debug: CMAKE_CURRENT_BINARY_DIR=${CMAKE_CURRENT_BINARY_DIR}")
+
+  configure_file (
+    "${CMAKE_CURRENT_SOURCE_DIR}/${kernel}/kernel.json.in"
+    "${CMAKE_CURRENT_BINARY_DIR}/${kernel}/kernel.json"
+    )
+  file(GLOB files "${CMAKE_CURRENT_SOURCE_DIR}/${kernel}/*.png")
+  foreach(file ${files})
+    configure_file(
+      "${file}"
+      "${CMAKE_CURRENT_BINARY_DIR}/${kernel}/"
+      COPYONLY
+      )
+  endforeach()
+  file(GLOB files "${CMAKE_CURRENT_SOURCE_DIR}/${kernel}/*.svg")
+  foreach(file ${files})
+    configure_file(
+      "${file}"
+      "${CMAKE_CURRENT_BINARY_DIR}/${kernel}/"
+      COPYONLY
+      )
+  endforeach()
+endfunction()
+
+file(GLOB _kernels LIST_DIRECTORIES true RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" "share/jupyter/kernels/*")
+foreach(_kernel IN LISTS _kernels)
+  message("Configure kernel: ${_kernel}")
+  configure_kernel("${_kernel}")
+endforeach()
+
 # Source files
 # ============
 
 set(XEUS_CPP_HEADERS
+    include/xeus-cpp/xbuffer.hpp
+    include/xeus-cpp/xholder.hpp
+    include/xeus-cpp/xmagics.hpp
+    include/xeus-cpp/xoptions.hpp
     include/xeus-cpp/xeus_cpp_config.hpp
     include/xeus-cpp/xinterpreter.hpp
+    include/xeus-cpp/xmanager.hpp
+    include/xeus-cpp/xpreamble.hpp
 )
 
 set(XEUS_CPP_SRC
@@ -144,10 +201,18 @@ set(XEUS_CPP_SRC
     src/xoptions.cpp
     src/xparser.cpp
     src/xutils.cpp
+    src/xdemangle.hpp
+    src/xinspect.hpp
+    src/xsystem.hpp
+    src/xparser.hpp
+    src/xmagics/os.hpp
+    src/xmagics/os.cpp
 )
 
 if(EMSCRIPTEN)
     list(APPEND XEUS_CPP_SRC src/xinterpreter_wasm.cpp)
+else()
+    list(APPEND XEUS_CPP_SRC src/xmagics/pythonexec.hpp src/xmagics/pythonexec.cpp)
 endif()
 
 set(XEUS_CPP_MAIN_SRC
@@ -161,7 +226,8 @@ include(CheckCXXCompilerFlag)
 
 string(TOUPPER "${CMAKE_BUILD_TYPE}" U_CMAKE_BUILD_TYPE)
 
-set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib; ${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}")
+set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib;${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}")
+list(REMOVE_DUPLICATES CMAKE_INSTALL_RPATH)
 
 macro(xeus_cpp_set_common_options target_name)
     if (MSVC)
@@ -215,7 +281,6 @@ macro(xeus_cpp_set_kernel_options target_name)
         find_package(Threads)
         target_link_libraries(${target_name} PRIVATE ${CMAKE_THREAD_LIBS_INIT})
     endif()
-    
 endmacro()
 
 # Common macro for shared and static library
@@ -251,7 +316,11 @@ macro(xeus_cpp_create_target target_name linkage output_name)
         set(XEUS_CPP_XEUS_TARGET xeus-static)
     endif ()
 
-    target_link_libraries(${target_name} PUBLIC ${XEUS_CPP_XEUS_TARGET} clangInterpreter pugixml argparse::argparse xtl)
+    target_link_libraries(${target_name} PUBLIC ${XEUS_CPP_XEUS_TARGET} clangCppInterOp pugixml argparse::argparse xtl)
+    if (NOT EMSCRIPTEN)
+      target_link_libraries(${target_name} PUBLIC Python::Python)
+    endif()
+
     if (WIN32 OR CYGWIN)
         #
     elseif (APPLE)
@@ -261,7 +330,7 @@ macro(xeus_cpp_create_target target_name linkage output_name)
         find_package(Threads) # TODO: add Threads as a dependence of xeus-static?
         target_link_libraries(${target_name} PRIVATE ${CMAKE_THREAD_LIBS_INIT})
     endif()
-    
+
 endmacro()
 
 # xeus-cpp-headers
@@ -308,6 +377,30 @@ if (XEUS_CPP_BUILD_EXECUTABLE)
     xeus_cpp_set_common_options(xcpp)
     xeus_cpp_set_kernel_options(xcpp)
     target_link_libraries(xcpp PRIVATE xeus-zmq)
+    set_target_properties(xcpp PROPERTIES
+      ENABLE_EXPORTS 1
+      CXX_STANDARD ${CMAKE_CXX_STANDARD})
+    target_link_libraries(xcpp PUBLIC xeus-cpp pthread Python::Python)
+
+    ##TODO: We may be need sse RPATH
+    set_target_properties(xcpp clangCppInterOp PROPERTIES INSTALL_RPATH_USE_LINK_PATH TRUE)
+    if(APPLE)
+      target_link_libraries(xcpp PUBLIC -Wl,-w -Wl,-bind_at_load -Wl,-undefined,dynamic_lookup)
+    elseif(NOT MSVC)
+      target_link_libraries(xcpp PUBLIC -Wl,--unresolved-symbols=ignore-in-object-files)
+    endif()
+
+    target_include_directories(xeus-cpp PUBLIC ${Python_INCLUDE_DIRS})
+    target_link_libraries(xeus-cpp PUBLIC ${PYTHON_LIBRARIES})
+    target_link_libraries(xeus-cpp ${PYTHON_LIBRARIES_Development_Main})
+    set_target_properties(xeus-cpp PROPERTIES
+                      PUBLIC_HEADER "${XEUS_CPP_HEADERS}"
+                      COMPILE_DEFINITIONS "XEUS_CPP_EXPORTS"
+                      PREFIX ""
+                      VERSION ${${PROJECT_NAME}_VERSION}
+                      SOVERSION ${XEUS_CPP_VERSION_MAJOR}
+                      OUTPUT_NAME "libxeus-cpp"
+                      CXX_STANDARD ${CMAKE_CXX_STANDARD})
 endif()
 
 if(EMSCRIPTEN)
@@ -337,10 +430,10 @@ set(XEUS_CPP_CMAKECONFIG_INSTALL_DIR "${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NA
 if (XEUS_CPP_BUILD_SHARED)
     install(TARGETS ${XEUS_CPP_TARGETS}
             EXPORT ${PROJECT_NAME}-targets
-            ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
-            LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
-            RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
-            PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/xeus-cpp)
+            ARCHIVE DESTINATION ${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}
+            LIBRARY DESTINATION ${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}
+            RUNTIME DESTINATION ${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_BINDIR}
+            PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_INCLUDEDIR}/xeus-cpp)
 
     # Makes the project importable from the build directory
     export(EXPORT ${PROJECT_NAME}-targets
@@ -350,15 +443,15 @@ endif ()
 # Install xcpp
 if (XEUS_CPP_BUILD_EXECUTABLE)
     install(TARGETS xcpp
-            RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR})
+            RUNTIME DESTINATION ${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_BINDIR})
 endif()
 
 if(XEUS_CPP_BUILD_EXECUTABLE OR EMSCRIPTEN)
     # Configuration and data directories for jupyter and xeus-cpp
     set(XJUPYTER_DATA_DIR "share/jupyter"    CACHE STRING "Jupyter data directory")
 
     # Install xcpp Jupyter kernelspec
-    set(KERNELSPEC_DIR ${CMAKE_CURRENT_SOURCE_DIR}/share/jupyter/kernels)
+    set(KERNELSPEC_DIR ${CMAKE_CURRENT_BINARY_DIR}/share/jupyter/kernels)
     install(DIRECTORY ${KERNELSPEC_DIR}
             DESTINATION ${XJUPYTER_DATA_DIR}
             PATTERN "*.in" EXCLUDE)
@@ -428,3 +521,7 @@ if(EMSCRIPTEN)
             "$<TARGET_FILE_DIR:xcpp>/xcpp.wasm"
             DESTINATION ${CMAKE_INSTALL_BINDIR})
 endif ()
+
+if(XEUS_CPP_INCLUDE_DOCS)
+    add_subdirectory(docs)
+endif()

environment-dev.yml

@@ -11,9 +11,9 @@ dependencies:
   - nlohmann_json
   - cppzmq
   - xtl
-  - clangdev >=16,<17
+  - CppInterOp
   - pugixml
-  - cpp-argparse
+  - cpp-argparse <3.0
   - zlib
   # Test dependencies
   - pytest

environment-wasm-host.yml

@@ -7,6 +7,6 @@ dependencies:
   - xeus-lite
   - xeus >=3.0.5,<4.0
   - xtl >=0.7,<0.8
-  - llvm =16.0.6
+  - CppInterOp
   - cpp-argparse
   - pugixml

include/xcpp/xdisplay.hpp

@@ -14,6 +14,8 @@
 
 #include "xcpp/xmime.hpp"
 
+#include "xeus/xinterpreter.hpp"
+
 namespace nl = nlohmann;
 
 namespace xcpp