diff --git a/.github/actions/install-deps/action.yaml b/.github/actions/install-deps/action.yaml index 833e11732f..d62ea00557 100644 --- a/.github/actions/install-deps/action.yaml +++ b/.github/actions/install-deps/action.yaml @@ -199,7 +199,10 @@ runs: - name: manylinux deps shell: bash - run: dnf install wget -y + run: | + if [ -x "$(command -v dnf)" ]; then + dnf install wget -y + fi if: ${{ runner.os == 'Linux' && inputs.cpp == 'true' && inputs.javascript == 'false' }} # TODO do this earlier? diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 629584c0b3..e592364c4d 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -190,18 +190,82 @@ jobs: - name: Python Build run: pnpm run build + if: ${{ !contains(matrix.os, 'windows') }} env: + PACKAGE: "perspective-python" + PSP_ARCH: ${{ matrix.arch }} + PSP_ROOT_DIR: ${{ github.workspace }} + PSP_BUILD_WHEEL: 1 + + - name: Python Build (Windows) + run: | + New-Item -ItemType Directory -Path $env:CARGO_TARGET_DIR -Force + pnpm run build + if: ${{ contains(matrix.os, 'windows') }} + env: + CARGO_TARGET_DIR: D:\psp-rust + PSP_ROOT_DIR: ${{ github.workspace }} VCPKG_ROOT: ${{ steps.init-step.outputs.VCPKG_INSTALLATION_ROOT }} PACKAGE: "perspective-python" - # PSP_USE_CCACHE: 1 PSP_ARCH: ${{ matrix.arch }} PSP_BUILD_WHEEL: 1 + # Windows sucks lol + - uses: actions/upload-artifact@v4 + if: ${{ runner.os == 'Windows' }} + with: + name: perspective-python-dist-${{ matrix.arch}}-${{ matrix.os }}-${{ matrix.python-version }} + path: D:\psp-rust\wheels\*.whl + - uses: actions/upload-artifact@v4 + # if: ${{ runner.os != 'Windows' }} with: name: perspective-python-dist-${{ matrix.arch}}-${{ matrix.os }}-${{ matrix.python-version }} path: rust/target/wheels/*.whl + build_emscripten_wheel: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: + - ubuntu-22.04 + arch: + - x86_64 + python-version: + - 3.9 + node-version: [20.x] + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Config + id: config-step + uses: ./.github/actions/config + + - name: Initialize Build + id: init-step + uses: ./.github/actions/install-deps + with: + javascript: "false" + arch: ${{ matrix.arch }} + manylinux: ${{ matrix.container && 'true' || 'false' }} + skip_cache: ${{ steps.config-step.outputs.SKIP_CACHE }} + + - name: Python Build Pyodide + run: pnpm install && pnpm run build + if: ${{ runner.os == 'Linux' && matrix.arch == 'x86_64' }} + env: + PSP_PYODIDE: 1 + PACKAGE: "perspective-python" + CI: 1 + + - uses: actions/upload-artifact@v4 + # if: ${{ runner.os != 'Windows' }} + with: + name: perspective-python-dist-wasm32-emscripten-${{ matrix.python-version }} + path: rust/target/wheels/*.whl + # ,-,---. . . . ,--,--' . # '|___/ . . . | ,-| ,-. ,-. ,-| `- | ,-. ,-. |- # ,| \ | | | | | | ,-| | | | | , | |-' `-. | diff --git a/.gitignore b/.gitignore index 232d2b879e..4c0b39f34b 100644 --- a/.gitignore +++ b/.gitignore @@ -238,6 +238,8 @@ rust/perspective/src/ts/ts-rs rust/perspective-viewer/src/ts/ts-rs rust/perspective-js/build rust/perspective-js/src/ts/ts-rs +rust/perspective-python/cpp +rust/perspective-python/cmake rust/perspective-server/cpp rust/perspective-server/cmake diff --git a/Cargo.lock b/Cargo.lock index a94973997c..43663ae5b4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1885,10 +1885,12 @@ name = "perspective-python" version = "3.0.0-rc.2" dependencies = [ "async-lock", + "cmake", + "cxx-build", "extend", "futures", + "num_cpus", "perspective-client", - "perspective-server", "pollster", "pyo3", "pyo3-build-config 0.20.3", diff --git a/Cargo.toml b/Cargo.toml index 797d2e37ae..5ae7fed518 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,7 +23,7 @@ members = [ "rust/perspective-js", "rust/perspective-python", "rust/perspective-server", - "examples/rust-axum", + "examples/rust-axum", ] [profile.dev] diff --git a/cpp/perspective/CMakeLists.txt b/cpp/perspective/CMakeLists.txt index 4c77b0e6da..eb964a2ddc 100644 --- a/cpp/perspective/CMakeLists.txt +++ b/cpp/perspective/CMakeLists.txt @@ -131,7 +131,7 @@ elseif(PSP_PYTHON_BUILD) if(NOT DEFINED PSP_PYTHON_VERSION) set(PSP_PYTHON_VERSION 3.10) endif() - if($ENV{PYODIDE}) + if(PSP_WASM_BUILD) set(PSP_PYODIDE 1) else() set(PSP_PYODIDE 0) @@ -223,14 +223,6 @@ if(NOT DEFINED PSP_WASM_EXCEPTIONS AND NOT PSP_PYTHON_BUILD) set(PSP_WASM_EXCEPTIONS ON) endif() -if(PSP_PYODIDE) - set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} \ - -s RELOCATABLE=1 \ - -s SIDE_MODULE=2 \ - ") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC") -endif() - # if(NOT WIN32) # set(CMAKE_CXX_FLAGS_RELEASE "-O3 -DNDEBUG") # set(CMAKE_C_FLAGS_RELEASE "-O3 -DNDEBUG") @@ -418,6 +410,14 @@ set(CMAKE_CXX_FLAGS " \ -O3 \ ") +if(PSP_PYODIDE) + set(RELOCATABLE_FLAGS "-sRELOCATABLE=1 -sSIDE_MODULE=2 -sWASM_BIGINT=1") + + string(APPEND CMAKE_EXE_LINKER_FLAGS "${RELOCATABLE_FLAGS}") + string(APPEND CMAKE_C_FLAGS "${RELOCATABLE_FLAGS}") + string(APPEND CMAKE_CXX_FLAGS "${RELOCATABLE_FLAGS}") +endif() + # Build header-only dependencies from external source psp_build_dep("date" "${PSP_CMAKE_MODULE_PATH}/date.txt.in") @@ -559,7 +559,7 @@ set(SOURCE_FILES ${PSP_CPP_SRC}/src/cpp/vocab.cpp ${PSP_CPP_SRC}/src/cpp/arrow_csv.cpp ${PSP_CPP_SRC}/src/cpp/server.cpp - ${PSP_CPP_SRC}/src/cpp/proto_api.cpp + ${PSP_CPP_SRC}/src/cpp/binding_api.cpp ) set(PYTHON_SOURCE_FILES ${SOURCE_FILES}) @@ -574,27 +574,36 @@ else() endif() # Common flags for WASM/JS build and Pyodide -set(PSP_WASM_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} \ - --no-entry \ - --closure=1 \ - -s NO_FILESYSTEM=1 \ - -s ALLOW_MEMORY_GROWTH=1 \ - -s MODULARIZE=1 \ - -s WASM_BIGINT=1 \ - -s INCOMING_MODULE_JS_API=locateFile,psp_heap_size,psp_stack_trace,HEAPU8,HEAPU32,instantiateWasm \ - -s TEXTDECODER=2 \ - -s STANDALONE_WASM=1 \ - -s DYNAMIC_EXECUTION=0 \ - -s BINARYEN_EXTRA_PASSES=--one-caller-inline-max-function-size=19306 \ - -s EXPORT_NAME=\"load_perspective\" \ - -s MAXIMUM_MEMORY=4gb \ - -s ERROR_ON_UNDEFINED_SYMBOLS=0 \ - -s NODEJS_CATCH_EXIT=0 \ - -s NODEJS_CATCH_REJECTION=0 \ - -s USE_ES6_IMPORT_META=1 \ - -s EXPORT_ES6=1 \ - -s EXPORTED_FUNCTIONS=_js_poll,_js_new_server,_js_free,_js_alloc,_js_handle_request,_js_new_session,_js_close_session \ -") +if(PSP_PYODIDE) + set(PSP_WASM_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} \ + --no-entry \ + -s EXPORTED_FUNCTIONS=_psp_poll,_psp_new_server,_psp_free,_psp_alloc,_psp_handle_request,_psp_new_session,_psp_close_session,_psp_delete_server \ + -s SIDE_MODULE=2 \ + ") +else() + set(PSP_WASM_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} \ + --no-entry \ + --closure=1 \ + -s NO_FILESYSTEM=1 \ + -s ALLOW_MEMORY_GROWTH=1 \ + -s MODULARIZE=1 \ + -s WASM_BIGINT=1 \ + -s INCOMING_MODULE_JS_API=locateFile,psp_heap_size,psp_stack_trace,HEAPU8,HEAPU32,instantiateWasm \ + -s TEXTDECODER=2 \ + -s STANDALONE_WASM=1 \ + -s DYNAMIC_EXECUTION=0 \ + -s BINARYEN_EXTRA_PASSES=--one-caller-inline-max-function-size=19306 \ + -s EXPORT_NAME=\"load_perspective\" \ + -s MAXIMUM_MEMORY=4gb \ + -s ERROR_ON_UNDEFINED_SYMBOLS=0 \ + -s NODEJS_CATCH_EXIT=0 \ + -s NODEJS_CATCH_REJECTION=0 \ + -s USE_ES6_IMPORT_META=1 \ + -s EXPORT_ES6=1 \ + -s EXPORTED_FUNCTIONS=_psp_poll,_psp_new_server,_psp_free,_psp_alloc,_psp_handle_request,_psp_new_session,_psp_close_session,_psp_delete_server \ + ") +endif() + if (PSP_WASM_EXCEPTIONS) set(PSP_WASM_LINKER_FLAGS "${PSP_WASM_LINKER_FLAGS} -s EXCEPTION_STACK_TRACES=1 ") endif() @@ -609,7 +618,7 @@ else() set(PSP_SANITIZE_FLAGS) endif() -if(PSP_WASM_BUILD) +if(PSP_WASM_BUILD AND NOT PSP_PYTHON_BUILD) set(CMAKE_EXE_LINKER_FLAGS "${PSP_WASM_LINKER_FLAGS} --pre-js \"${PSP_CPP_SRC}/env.js\" ") add_library(psp ${WASM_SOURCE_FILES}) @@ -617,7 +626,7 @@ if(PSP_WASM_BUILD) set_target_properties(psp PROPERTIES COMPILE_FLAGS "") target_link_libraries(psp PRIVATE arrow re2 protos) - add_executable(perspective_esm src/cpp/emscripten_api.cpp) + add_executable(perspective_esm src/cpp/binding_api.cpp) target_link_libraries(perspective_esm psp protos) target_compile_definitions(perspective_esm PRIVATE PSP_ENABLE_WASM=1) target_link_options(perspective_esm PUBLIC -sENVIRONMENT="web" ${PSP_SANITIZE_FLAGS}) @@ -644,25 +653,25 @@ elseif(PSP_CPP_BUILD OR PSP_PYTHON_BUILD) set(CMAKE_SHARED_LIBRARY_PREFIX lib) endif() - if(PSP_PYODIDE) - add_library(psppy SHARED) - set(PSP_PYTHON_DEFS PSP_ENABLE_WASM=1) - target_compile_definitions(psppy PRIVATE ${PSP_PYTHON_DEFS}) - include_directories(${PSP_PYTHON_SRC}/include) - target_compile_definitions(psppy PRIVATE PSP_ENABLE_PYTHON=1) - target_compile_options(psppy PRIVATE -fvisibility=hidden) - target_link_libraries(psppy arrow re2) - add_custom_command(TARGET psppy POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy $ ${PSP_PYTHON_SRC}/table/) - elseif(PSP_PYTHON_BUILD) + if(PSP_PYTHON_BUILD) # ####################### # Python extra targets # # ####################### - add_library(psp STATIC ${PYTHON_SOURCE_FILES}) + if(PSP_WASM_BUILD) + set(CMAKE_EXECUTABLE_SUFFIX ".wasm") + set(CMAKE_EXE_LINKER_FLAGS "${PSP_WASM_LINKER_FLAGS} --pre-js \"${PSP_CPP_SRC}/env.js\" ") + add_library(psp ${PYTHON_SOURCE_FILES}) + add_executable(psppy ${PSP_CPP_SRC}/src/cpp/binding_api.cpp) + target_compile_definitions(psppy PRIVATE PSP_ENABLE_PYTHON=1 PSP_ENABLE_WASM=1) + target_link_libraries(psppy PRIVATE psp protos) + else() + add_library(psp SHARED ${PYTHON_SOURCE_FILES}) + target_compile_definitions(psp PRIVATE PSP_ENABLE_PYTHON=1 PSP_PARALLEL_FOR=1) + endif() # add_library(psppy SHARED ${PYTHON_BINDING_SOURCE_FILES}) include_directories(${PSP_PYTHON_SRC}/include) - target_compile_definitions(psp PRIVATE PSP_ENABLE_PYTHON=1 PSP_PARALLEL_FOR=1) # target_compile_definitions(psppy PRIVATE PSP_ENABLE_PYTHON=1 PSP_PARALLEL_FOR=1) if(WIN32) @@ -678,7 +687,9 @@ elseif(PSP_CPP_BUILD OR PSP_PYTHON_BUILD) target_compile_options(psp PRIVATE -fvisibility=hidden) # target_compile_options(psppy PRIVATE -fvisibility=hidden) + elseif(MANYLINUX) else() + target_compile_options(psp PRIVATE -fvisibility=hidden) # target_compile_options(psppy PRIVATE -Wdeprecated-declarations) endif() diff --git a/cpp/perspective/src/cpp/binding_api.cpp b/cpp/perspective/src/cpp/binding_api.cpp new file mode 100644 index 0000000000..709d2d0f6c --- /dev/null +++ b/cpp/perspective/src/cpp/binding_api.cpp @@ -0,0 +1,122 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +#include "perspective/exports.h" +#include "perspective/server.h" +#include +#include +#include +#include +#include + +using namespace perspective::server; + +#pragma pack(push, 1) +struct EncodedApiResp { + void* data; + std::uint32_t size; + std::uint32_t client_id; +}; + +struct EncodedApiEntries { + std::uint32_t size; + EncodedApiResp* entries; +}; +#pragma pack(pop) + +void +encode_api_response( + const ProtoServerResp& msg, EncodedApiResp* encoded +) { + auto* data = new char[msg.data.size()]; + std::copy(msg.data.begin(), msg.data.end(), data); + + encoded->data = data; + encoded->size = msg.data.size(); + encoded->client_id = msg.client_id; +} + +EncodedApiEntries* +encode_api_responses(const std::vector>& msgs) { + auto* encoded = new EncodedApiEntries; + encoded->entries = new EncodedApiResp[msgs.size()]; + + encoded->size = msgs.size(); + auto* encoded_mem = encoded->entries; + for (int i = 0; i < msgs.size(); i++) { + encode_api_response(msgs[i], &encoded_mem[i]); + } + + return encoded; +} + +extern "C" { + +PERSPECTIVE_EXPORT +ProtoServer* +psp_new_server() { + return new ProtoServer; +} + +PERSPECTIVE_EXPORT +EncodedApiEntries* +psp_handle_request( + ProtoServer* server, + std::uint32_t client_id, + char* msg_ptr, + std::size_t msg_len +) { + std::string msg(msg_ptr, msg_len); + auto msgs = server->handle_request(client_id, msg); + return encode_api_responses(msgs); +} + +PERSPECTIVE_EXPORT +EncodedApiEntries* +psp_poll(ProtoServer* server) { + auto responses = server->poll(); + return encode_api_responses(responses); +} + +PERSPECTIVE_EXPORT +std::uint32_t +psp_new_session(ProtoServer* server) { + return server->new_session(); +} + +PERSPECTIVE_EXPORT +void +psp_close_session(ProtoServer* server, std::uint32_t client_id) { + server->close_session(client_id); +} + +PERSPECTIVE_EXPORT +std::size_t +psp_alloc(std::size_t size) { + auto* mem = (char*)malloc(size); + return (size_t)mem; +} + +PERSPECTIVE_EXPORT +void +psp_free(void* ptr) { + free(ptr); +} + +PERSPECTIVE_EXPORT +void +psp_delete_server(void* proto_server) { + auto* server = (ProtoServer*)proto_server; + delete server; +} + +} // end extern "C" diff --git a/cpp/perspective/src/cpp/emscripten_api.cpp b/cpp/perspective/src/cpp/emscripten_api.cpp deleted file mode 100644 index 8bcb201e60..0000000000 --- a/cpp/perspective/src/cpp/emscripten_api.cpp +++ /dev/null @@ -1,156 +0,0 @@ -// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ -// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ -// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ -// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ -// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ -// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ -// ┃ Copyright (c) 2017, the Perspective Authors. ┃ -// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ -// ┃ This file is part of the Perspective library, distributed under the terms ┃ -// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ -// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ - -#include "perspective/exports.h" -#include -#include -#include -#include -#include - -// TODO: use uintptr_t instead of size_t. - -/// places a 32 bit number at the first 4 bytes of the given address -void -place_number(std::uint32_t num, std::uint8_t* addr) { - addr[0] = num & 0x000000ff; - addr[1] = (num & 0x0000ff00) >> 8; - addr[2] = (num & 0x00ff0000) >> 16; - addr[3] = (num & 0xff000000) >> 24; - // std::copy(reinterpret_cast(num), - // reinterpret_cast(num + sizeof(num)), addr); -} - -/// copies the given vector into a new memory region that is encoded -/// for easy reading -size_t -encode_vec(std::vector vec) { - // TODO: this is 32-bit only! wasm64 beware - auto* new_ptr = (std::uint32_t*)malloc(vec.size() + sizeof(std::uint32_t)); - *new_ptr = vec.size(); - // place_number(size, new_ptr); - std::copy(vec.begin(), vec.end(), (std::uint8_t*)(new_ptr + 1)); - return (size_t)new_ptr; -} - -size_t -encode_string(std::string s) { - std::vector vec(s.begin(), s.end()); - return encode_vec(vec); -} - -// struct EncodedApiResp { -// size_t data; -// std::uint32_t size; -// std::uint32_t client_id; -// }; - -/// The size_t is a pointer that points to an object: -/// [data_ptr,client_id] -/// The string (data) is a (len,data) tuple. -/// The string is copied and leaked and so is the pointer here. -/// TODO: we need to make a lot of complex `free`ing procedures. -void -encode_api_response(const ProtoApiResponse& msg, std::uint32_t* encoded) { - // size_t data = encode_string(msg.data); - - // auto* new_ptr = (std::uint32_t*)malloc(vec.size() + - // sizeof(std::uint32_t)); - // TODO: this is 32-bit only! wasm64 beware - auto* data = (std::uint32_t*)malloc(msg.data.size()); - // *new_ptr = msg.data.size(); - // place_number(size, new_ptr); - std::copy(msg.data.begin(), msg.data.end(), (std::uint8_t*)(data)); - - // EncodedApiResp - // encoded; // = (EncodedApiResp*)malloc(sizeof(EncodedApiResp)); - *encoded = (size_t)data; - *(encoded + 1) = msg.data.size(); - *(encoded + 2) = msg.client_id; -} - -size_t -encode_api_responses(const std::vector& msgs) { - auto* encoded = (std::uint32_t*)malloc( - msgs.size() * sizeof(std::uint32_t) * 3 + sizeof(std::uint32_t) - ); - - *encoded = msgs.size(); - auto* encoded_mem = (std::uint32_t*)(encoded + 1); - for (int i = 0; i < msgs.size(); i++) { - encode_api_response(msgs[i], (encoded_mem + i * 3)); - } - - return (size_t)encoded; -} - -extern "C" { - -PERSPECTIVE_EXPORT -ProtoApiServer* -js_new_server() { - auto* server = new ProtoApiServer; - return server; -} - -PERSPECTIVE_EXPORT -size_t -js_handle_request( - ProtoApiServer* server, - std::uint32_t client_id, - char* msg_ptr, - std::size_t msg_len -) { - std::string msg(msg_ptr, msg_len); - auto msgs = server->handle_request(client_id, msg); - return encode_api_responses(msgs); -} - -PERSPECTIVE_EXPORT -size_t -js_poll(ProtoApiServer* server) { - auto responses = server->poll(); - return encode_api_responses(responses); -} - -PERSPECTIVE_EXPORT -std::uint32_t -js_new_session(ProtoApiServer* server) { - return server->new_session(); -} - -PERSPECTIVE_EXPORT -void -js_close_session(ProtoApiServer* server, std::uint32_t client_id) { - server->close_session(client_id); -} - -PERSPECTIVE_EXPORT -std::size_t -js_alloc(std::size_t size) { - auto* mem = (char*)malloc(size); - return (size_t)mem; -} - -PERSPECTIVE_EXPORT -void -js_free(void* ptr) { - free(ptr); -} - -} // end extern "C" - -void -js_delete_server(void* proto_server) { - auto* server = (ProtoApiServer*)proto_server; - delete server; -} diff --git a/cpp/perspective/src/cpp/gnode.cpp b/cpp/perspective/src/cpp/gnode.cpp index 3113cbddac..4a511f7232 100644 --- a/cpp/perspective/src/cpp/gnode.cpp +++ b/cpp/perspective/src/cpp/gnode.cpp @@ -186,11 +186,13 @@ t_gnode::calc_transition( if (!row_pre_existed && !cur_valid && !t_env::backout_invalid_neq_ft()) { trans = VALUE_TRANSITION_NEQ_FT; - } else if (row_pre_existed && !prev_valid && !cur_valid && !t_env::backout_eq_invalid_invalid()) { + } else if (row_pre_existed && !prev_valid && !cur_valid + && !t_env::backout_eq_invalid_invalid()) { trans = VALUE_TRANSITION_EQ_TT; } else if (!prev_existed && !exists) { trans = VALUE_TRANSITION_EQ_FF; - } else if (row_pre_existed && exists && !prev_valid && cur_valid && !t_env::backout_nveq_ft()) { + } else if (row_pre_existed && exists && !prev_valid && cur_valid + && !t_env::backout_nveq_ft()) { trans = VALUE_TRANSITION_NVEQ_FT; } else if (prev_existed && exists && prev_cur_eq) { trans = VALUE_TRANSITION_EQ_TT; diff --git a/cpp/perspective/src/cpp/proto_api.cpp b/cpp/perspective/src/cpp/proto_api.cpp deleted file mode 100644 index 9e5a89000f..0000000000 --- a/cpp/perspective/src/cpp/proto_api.cpp +++ /dev/null @@ -1,68 +0,0 @@ -// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ -// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ -// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ -// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ -// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ -// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ -// ┃ Copyright (c) 2017, the Perspective Authors. ┃ -// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ -// ┃ This file is part of the Perspective library, distributed under the terms ┃ -// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ -// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ - -#include "perspective/server.h" -#include "perspective/proto_api.h" -#include - -class ProtoApiServer::ProtoApiServerImpl { -public: - std::unique_ptr m_server; - ProtoApiServerImpl(); - ~ProtoApiServerImpl(); -}; - -ProtoApiServer::ProtoApiServer() : - m_impl(std::make_unique()) {} -ProtoApiServer::~ProtoApiServer() = default; - -ProtoApiServer::ProtoApiServerImpl::ProtoApiServerImpl() : - m_server(std::make_unique()) {} -ProtoApiServer::ProtoApiServerImpl::~ProtoApiServerImpl() = default; - -std::uint32_t -ProtoApiServer::new_session() { - return m_impl->m_server->new_session(); -} - -void -ProtoApiServer::close_session(const std::uint32_t& client_id) { - return m_impl->m_server->close_session(client_id); -} - -std::vector -ProtoApiServer::handle_request(std::uint32_t client_id, const std::string& data) - const { - auto responses = m_impl->m_server->handle_request(client_id, data); - std::vector results; - for (const auto& msg : responses) { - ProtoApiResponse resp; - resp.client_id = msg.client_id; - resp.data = msg.data; - results.push_back(resp); - } - - return results; -} - -std::vector -ProtoApiServer::poll() { - std::vector results; - for (const auto& msg : m_impl->m_server->poll()) { - ProtoApiResponse resp; - resp.client_id = msg.client_id; - resp.data = msg.data; - results.push_back(resp); - } - - return results; -} diff --git a/cpp/perspective/src/cpp/server.cpp b/cpp/perspective/src/cpp/server.cpp index 1670a81536..719a25a23f 100644 --- a/cpp/perspective/src/cpp/server.cpp +++ b/cpp/perspective/src/cpp/server.cpp @@ -807,7 +807,7 @@ parse_format_options( viewport.has_end_row() ? viewport.end_row() : (viewport_height != 0 ? out.start_row + viewport_height : max_rows - ) + ) ); out.end_col = std::min( max_cols, diff --git a/cpp/perspective/src/cpp/sparse_tree.cpp b/cpp/perspective/src/cpp/sparse_tree.cpp index 2244346f52..812c64409e 100644 --- a/cpp/perspective/src/cpp/sparse_tree.cpp +++ b/cpp/perspective/src/cpp/sparse_tree.cpp @@ -2259,7 +2259,8 @@ t_stree::get_aggregates_for_sorting( auto which_agg = agg_indices[idx]; if (which_agg < 0) { aggregates[idx] = get_sortby_value(nidx); - } else if ((ctx2 != nullptr) || (size_t(which_agg) >= m_aggcols.size())) { + } else if ((ctx2 != nullptr) + || (size_t(which_agg) >= m_aggcols.size())) { aggregates[idx].set(t_none()); if (ctx2 != nullptr) { if ((ctx2->get_config().get_totals() == TOTALS_BEFORE) diff --git a/cpp/perspective/src/cpp/table.cpp b/cpp/perspective/src/cpp/table.cpp index 103a7acf15..031b9f44b5 100644 --- a/cpp/perspective/src/cpp/table.cpp +++ b/cpp/perspective/src/cpp/table.cpp @@ -522,7 +522,9 @@ PROMOTE_IMPL(DTYPE_INT32, DTYPE_INT64, DTYPE_INT64) template static A json_into(const rapidjson::Value& value) { - if constexpr (std::is_same_v || std::is_same_v || std::is_same_v) { + if constexpr (std::is_same_v + || std::is_same_v + || std::is_same_v) { if (value.IsInt()) { return value.GetInt(); } @@ -540,7 +542,8 @@ json_into(const rapidjson::Value& value) { return std::atoi(value.GetString()); } else if constexpr (std::is_same_v) { return std::atoll(value.GetString()); - } else if constexpr (std::is_same_v || std::is_same_v) { + } else if constexpr (std::is_same_v + || std::is_same_v) { return std::atof(value.GetString()); } else { static_assert(!std::is_same_v, "No coercion for type"); @@ -762,9 +765,11 @@ fill_column_json( case t_dtype::DTYPE_BOOL: { if (value.IsBool()) [[likely]] { col->set_nth(i, value.GetBool()); - } else if (value.IsString() && istrequals(value.GetString(), "true")) { + } else if (value.IsString() + && istrequals(value.GetString(), "true")) { col->set_nth(i, true); - } else if (value.IsString() && istrequals(value.GetString(), "false")) { + } else if (value.IsString() + && istrequals(value.GetString(), "false")) { col->set_nth(i, false); } else if (value.IsInt()) { col->set_nth(i, value.GetInt() != 0); diff --git a/cpp/perspective/src/cpp/view_config.cpp b/cpp/perspective/src/cpp/view_config.cpp index e87358d7b7..ed3daaff06 100644 --- a/cpp/perspective/src/cpp/view_config.cpp +++ b/cpp/perspective/src/cpp/view_config.cpp @@ -332,7 +332,8 @@ t_view_config::fill_aggspecs(const std::shared_ptr& schema) { if (is_column_only) { // Always sort by `ANY` in column only views agg_type = t_aggtype::AGGTYPE_ANY; - } else if ((is_row_pivot && is_row_sort) || (is_column_pivot && !is_row_sort)) { + } else if ((is_row_pivot && is_row_sort) + || (is_column_pivot && !is_row_sort)) { // Otherwise if the hidden column is in pivot on the same axis, // use `UNIQUE` agg_type = t_aggtype::AGGTYPE_UNIQUE; diff --git a/cpp/perspective/src/include/perspective/exprtk.h b/cpp/perspective/src/include/perspective/exprtk.h index 2889e4ab49..bd3fce728f 100644 --- a/cpp/perspective/src/include/perspective/exprtk.h +++ b/cpp/perspective/src/include/perspective/exprtk.h @@ -148,7 +148,7 @@ namespace details { // #endif } // end namespace details - } // end namespace numeric + } // end namespace numeric inline bool is_true(const t_tscalar& v); @@ -1219,7 +1219,7 @@ namespace details { } } // end namespace details - } // end namespace numeric + } // end namespace numeric /** * Override the comparison operators to return a `t_tscalar` of DTYPE_BOOL diff --git a/cpp/perspective/src/include/perspective/proto_api.h b/cpp/perspective/src/include/perspective/proto_api.h deleted file mode 100644 index 951cb772a7..0000000000 --- a/cpp/perspective/src/include/perspective/proto_api.h +++ /dev/null @@ -1,46 +0,0 @@ -// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ -// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ -// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ -// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ -// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ -// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ -// ┃ Copyright (c) 2017, the Perspective Authors. ┃ -// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ -// ┃ This file is part of the Perspective library, distributed under the terms ┃ -// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ -// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ - -#pragma once - -#include -#include -#include - -#if defined(_WIN32) || defined(WIN32) -#define PERSPECTIVE_EXPORT __declspec(dllexport) -#else -#define PERSPECTIVE_EXPORT __attribute__((visibility("default"))) -#endif - -struct ProtoApiResponse { - std::string data; - std::uint32_t client_id; -}; - -class PERSPECTIVE_EXPORT ProtoApiServer { -private: - class ProtoApiServerImpl; - std::unique_ptr m_impl; - -public: - ProtoApiServer(); - ~ProtoApiServer(); - - std::uint32_t new_session(); - void close_session(const std::uint32_t& client_id); - - std::vector - handle_request(std::uint32_t client_id, const std::string& data) const; - - std::vector poll(); -}; \ No newline at end of file diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index bc23878168..7d29ba7c33 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -13,7 +13,7 @@ importers: version: 2.11.2 fast-xml-parser: specifier: ^4.4.0 - version: 4.4.0 + version: 4.4.1 gif-encoder-2: specifier: ^1.0.5 version: 1.0.5 @@ -5046,8 +5046,8 @@ packages: fast-url-parser@1.1.3: resolution: {integrity: sha512-5jOCVXADYNuRkKFzNJ0dCCewsZiYo0dz8QNYljkOpFC6r2U4OBmKtvm/Tsuh4w1YYdDqDb31a8TVhBJ2OJKdqQ==} - fast-xml-parser@4.4.0: - resolution: {integrity: sha512-kLY3jFlwIYwBNDojclKsNAC12sfD6NwW74QB2CoNGPvtVxjliYehVunB3HYyNi+n4Tt1dAcgwYvmKF/Z18flqg==} + fast-xml-parser@4.4.1: + resolution: {integrity: sha512-xkjOecfnKGkSsOwtZ5Pz7Us/T6mrbPQrq0nh+aCO5V9nk5NLWmasAHumTKjiPJPWANe+kAZ84Jc8ooJkzZ88Sw==} hasBin: true fastest-levenshtein@1.0.16: @@ -15083,7 +15083,7 @@ snapshots: dependencies: punycode: 1.4.1 - fast-xml-parser@4.4.0: + fast-xml-parser@4.4.1: dependencies: strnum: 1.0.5 diff --git a/rust-toolchain.toml b/rust-toolchain.toml index a2dc41062a..0c2fc7a56c 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -13,4 +13,4 @@ [toolchain] channel = "nightly-2024-05-07" components = ["rustfmt", "clippy", "rust-src"] -targets = ["wasm32-unknown-unknown"] +targets = ["wasm32-unknown-unknown", "wasm32-unknown-emscripten"] diff --git a/rust/perspective-client/Cargo.toml b/rust/perspective-client/Cargo.toml index 003f81c95d..14601d8aaa 100644 --- a/rust/perspective-client/Cargo.toml +++ b/rust/perspective-client/Cargo.toml @@ -36,6 +36,7 @@ rustdoc-args = ["--html-in-header", "docs/index.html"] [features] default = [] external-proto = ["protobuf-src"] +external-protoc = [] [lib] crate-type = ["rlib"] @@ -50,12 +51,12 @@ async-lock = { version = "2.5.0" } futures = { version = "0.3.28" } itertools = { version = "0.10.1" } nanoid = { version = "0.4.0" } -paste = { version = "1.0.14" } +paste = { version = "1.0.12" } prost-types = { version = "0.12.3" } serde = { version = "1.0", features = ["derive"] } serde_bytes = { version = "0.11" } serde_json = { version = "1.0.107", features = ["raw_value"] } -thiserror = { version = "1.0.56" } +thiserror = { version = "1.0.55" } tracing = { version = ">=0.1.36" } tracing-unwrap = "1.0.1" diff --git a/rust/perspective-js/Cargo.toml b/rust/perspective-js/Cargo.toml index 158be226c4..50a10979d1 100644 --- a/rust/perspective-js/Cargo.toml +++ b/rust/perspective-js/Cargo.toml @@ -77,7 +77,7 @@ tracing = { version = ">=0.1.36" } tracing-subscriber = "0.3.15" console_error_panic_hook = "0.1.6" wasm-bindgen = { version = "=0.2.92", features = ["serde-serialize"] } -wasm-bindgen-futures = "0.4.42" +wasm-bindgen-futures = "0.4.41" [dependencies.web-sys] version = "0.3.64" diff --git a/rust/perspective-js/src/ts/emscripten_api.ts b/rust/perspective-js/src/ts/emscripten_api.ts index a1387ab252..c39c399941 100644 --- a/rust/perspective-js/src/ts/emscripten_api.ts +++ b/rust/perspective-js/src/ts/emscripten_api.ts @@ -24,19 +24,19 @@ export interface EmscriptenApi { HEAPU16: Uint16Array; HEAP32: Int32Array; HEAPU32: Uint32Array; - _js_alloc(size: number): number; - _js_free(ptr: number): void; - _js_new_server(): EmscriptenServer; - _js_delete_server(server: EmscriptenServer): void; - _js_handle_request( + _psp_alloc(size: number): number; + _psp_free(ptr: number): void; + _psp_new_server(): EmscriptenServer; + _psp_delete_server(server: EmscriptenServer): void; + _psp_handle_request( server: EmscriptenServer, client_id: number, buffer_ptr: number, buffer_len: number ): number; - _js_poll(server: EmscriptenServer): number; - _js_new_session(server: EmscriptenServer): number; - _js_close_session(server: EmscriptenServer, client_id: number): void; + _psp_poll(server: EmscriptenServer): number; + _psp_new_session(server: EmscriptenServer): number; + _psp_close_session(server: EmscriptenServer, client_id: number): void; } export async function compile_perspective( @@ -57,7 +57,7 @@ export async function compile_perspective( const str = Error().stack || ""; const textEncoder = new TextEncoder(); const bytes = textEncoder.encode(str); - const ptr = module._js_alloc(bytes.byteLength + 1); + const ptr = module._psp_js_alloc(bytes.byteLength + 1); module.HEAPU8.set(bytes, ptr); module.HEAPU8[ptr + bytes.byteLength] = 0; return ptr; diff --git a/rust/perspective-js/src/ts/engine.ts b/rust/perspective-js/src/ts/engine.ts index 5d3c0b68b4..01aa61c2ac 100644 --- a/rust/perspective-js/src/ts/engine.ts +++ b/rust/perspective-js/src/ts/engine.ts @@ -26,7 +26,7 @@ export class PerspectiveServer { this.clients = new Map(); this.id_gen = 0; this.module = module; - this.server = module._js_new_server(); + this.server = module._psp_new_server(); } /** @@ -35,7 +35,7 @@ export class PerspectiveServer { make_session( callback: (buffer: Uint8Array) => Promise ): PerspectiveSession { - const client_id = this.module._js_new_session(this.server); + const client_id = this.module._psp_new_session(this.server); this.clients.set(client_id, callback); return new PerspectiveSession( this.module, @@ -46,7 +46,7 @@ export class PerspectiveServer { } delete() { - this.module._js_delete_server(this.server); + this.module._psp_delete_server(this.server); } } @@ -55,7 +55,7 @@ export class PerspectiveSession { private mod: EmscriptenApi, private server: EmscriptenServer, private client_id: number, - private client_map: Map void> + private client_map: Map Promise> ) {} async handle_request(view: Uint8Array) { @@ -63,7 +63,7 @@ export class PerspectiveSession { this.mod, view, async (viewPtr) => { - return this.mod._js_handle_request( + return this.mod._psp_handle_request( this.server, this.client_id, viewPtr, @@ -72,20 +72,20 @@ export class PerspectiveSession { } ); - decode_api_responses(this.mod, ptr, async (msg: ApiResponse) => { + await decode_api_responses(this.mod, ptr, async (msg: ApiResponse) => { await this.client_map.get(msg.client_id)!(msg.data); }); } poll() { - const polled = this.mod._js_poll(this.server); + const polled = this.mod._psp_poll(this.server); decode_api_responses(this.mod, polled, async (msg: ApiResponse) => { await this.client_map.get(msg.client_id)!(msg.data); }); } close() { - this.mod._js_close_session(this.server, this.client_id); + this.mod._psp_close_session(this.server, this.client_id); } } @@ -94,16 +94,17 @@ async function convert_typed_array_to_pointer( array: Uint8Array, callback: (_: number) => Promise ): Promise { - const ptr = core._js_alloc(array.byteLength); + const ptr = core._psp_alloc(array.byteLength); core.HEAPU8.set(array, ptr); const msg = await callback(ptr); - core._js_free(ptr); + core._psp_free(ptr); return msg; } function convert_pointer_to_u32_array(core: EmscriptenApi, ptr: number) { const len = core.HEAPU32[ptr >>> 2]; - return new Uint32Array(core.HEAPU8.buffer, ptr + 4, len * 3); + const data_ptr = core.HEAPU32[(ptr >>> 2) + 1]; + return new Uint32Array(core.HEAPU8.buffer, data_ptr, len * 3); } /** @@ -136,15 +137,22 @@ async function decode_api_responses( callback: (_: ApiResponse) => Promise ) { const responses = convert_pointer_to_u32_array(core, ptr); - for (let i = 0; i < responses.length / 3; i++) { - const data_ptr = responses[i * 3]; - const length = responses[i * 3 + 1]; - const client_id = responses[i * 3 + 2]; - const data = new Uint8Array(core.HEAPU8.buffer, data_ptr, length); - const resp = { client_id, data }; - await callback(resp); - core._js_free(data_ptr); - } + try { + for (let i = 0; i < responses.length / 3; i++) { + const data_ptr = responses[i * 3]; + const length = responses[i * 3 + 1]; + const client_id = responses[i * 3 + 2]; + const data = new Uint8Array(core.HEAPU8.buffer, data_ptr, length); + const resp = { client_id, data }; + await callback(resp); + } + } finally { + for (let i = 0; i < responses.length / 3; i++) { + const data_ptr = responses[i * 3]; + core._psp_free(data_ptr); + } - core._js_free(ptr); + core._psp_free(responses.byteOffset); + core._psp_free(ptr); + } } diff --git a/rust/perspective-js/src/ts/perspective.ts b/rust/perspective-js/src/ts/perspective.ts index 26b864b624..95fecec7c2 100644 --- a/rust/perspective-js/src/ts/perspective.ts +++ b/rust/perspective-js/src/ts/perspective.ts @@ -20,7 +20,7 @@ type WasmElement = { __wasm_module__: Promise; }; -async function get_module() { +export async function compile_perspective() { let elem = customElements.get( "perspective-viewer" ) as unknown as WasmElement; @@ -36,12 +36,12 @@ async function get_module() { } export async function websocket(url: string | URL) { - const wasm_module = get_module(); + const wasm_module = compile_perspective(); return await api.websocket(wasm_module, url); } export async function worker() { - const wasm_module = get_module(); + const wasm_module = compile_perspective(); return await api.worker.call(undefined, wasm_module); } diff --git a/rust/perspective-python/Cargo.toml b/rust/perspective-python/Cargo.toml index 0529090f7d..d0479056d0 100644 --- a/rust/perspective-python/Cargo.toml +++ b/rust/perspective-python/Cargo.toml @@ -20,16 +20,18 @@ license = "Apache-2.0" homepage = "https://perspective.finos.org" authors = ["Andrew Stein "] keywords = [] -build = "build.rs" +build = "build/main.rs" include = [ "bench/**/*.py", - "./build.rs", + "build/**/*", "./Cargo.toml", "./package.json", "perspective/**/*.py", "./pyproject.toml", "src/**/*.rs", "docs/**/*", + "cpp/**/*", + "cmake/**/*", ] [package.metadata.docs.rs] @@ -37,22 +39,23 @@ rustdoc-args = ["--html-in-header", "docs/index.html"] [features] default = [] -external-cpp = [ - "perspective-server/external-cpp", - "perspective-client/external-proto", -] +sdist = [] +external-cpp = ["perspective-client/external-proto"] [lib] crate-type = ["cdylib"] [build-dependencies] +cxx-build = "1.0.115" +cmake = "0.1.50" +num_cpus = "^1.16.0" pyo3-build-config = "0.20.2" python-config-rs = "0.1.2" [dependencies] async-lock = "2.5.0" perspective-client = { version = "3.0.0-rc.2", path = "../perspective-client" } -perspective-server = { version = "3.0.0-rc.2", path = "../perspective-server" } +# perspective-server = { version = "3.0.0-rc.2", path = "../perspective-server" } pollster = "0.3.0" extend = "1.1.2" futures = "0.3.28" diff --git a/rust/perspective-python/build.mjs b/rust/perspective-python/build.mjs index 9cea709406..16a17cb7bd 100644 --- a/rust/perspective-python/build.mjs +++ b/rust/perspective-python/build.mjs @@ -10,22 +10,20 @@ // ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ // ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ -import { execSync } from "child_process"; import * as fs from "node:fs"; import pkg from "./package.json" assert { type: "json" }; +import sh from "../../tools/perspective-scripts/sh.mjs"; +import * as url from "url"; + +const __dirname = url.fileURLToPath(new URL(".", import.meta.url)).slice(0, -1); let flags = "--release"; if (!!process.env.PSP_DEBUG) { flags = ""; } -const opts = { - stdio: "inherit", - env: { - ...process.env, - PSP_ROOT_DIR: "../..", - }, -}; +const python_version = process.env.PSP_PYTHON_VERSION || "3.12"; +const is_pyodide = !!process.env.PSP_PYODIDE; const version = pkg.version.replace(/-(rc|alpha|beta)\.\d+/, (x) => x.replace("-", "").replace(".", "") @@ -33,12 +31,36 @@ const version = pkg.version.replace(/-(rc|alpha|beta)\.\d+/, (x) => fs.mkdirSync(`./perspective_python-${version}.data`, { recursive: true }); -const build_wheel = !!process.env.PSP_BUILD_WHEEL; +const cwd = process.cwd(); +const cmd = sh(); + +if (is_pyodide) { + const emsdkdir = sh.path`${__dirname}/../../.emsdk`; + const { emscripten } = JSON.parse( + fs.readFileSync(sh.path`${__dirname}/../../package.json`) + ); + cmd.sh`cd ${emsdkdir}`.sh`. ./emsdk_env.sh` + .sh`./emsdk activate ${emscripten}`.sh`cd ${cwd}`; +} + +// if not windows +if (process.platform !== "win32") { + cmd.env({ + PSP_ROOT_DIR: "../..", + }); +} + +const build_wheel = !!process.env.PSP_BUILD_WHEEL || is_pyodide; const build_sdist = !!process.env.PSP_BUILD_SDIST; if (build_wheel) { let target = ""; - if (process.env.PSP_ARCH === "x86_64" && process.platform === "darwin") { + if (is_pyodide) { + target = `--target=wasm32-unknown-emscripten -i${python_version}`; + } else if ( + process.env.PSP_ARCH === "x86_64" && + process.platform === "darwin" + ) { target = "--target=x86_64-apple-darwin"; } else if ( process.env.PSP_ARCH === "aarch64" && @@ -58,13 +80,15 @@ if (build_wheel) { target = "--target=aarch64-unknown-linux-gnu"; } - execSync(`maturin build ${flags} --features=external-cpp ${target}`, opts); + cmd.sh(`maturin build ${flags} --features=external-cpp ${target}`); } if (build_sdist) { - execSync(`maturin sdist`, opts); + cmd.sh(`maturin sdist`); } if (!build_wheel && !build_sdist) { - execSync(`maturin develop ${flags} --features=external-cpp`, opts); + cmd.sh(`maturin develop ${flags} --features=external-cpp`); } + +cmd.runSync(); diff --git a/rust/perspective-python/build/main.rs b/rust/perspective-python/build/main.rs new file mode 100644 index 0000000000..5debc8ab53 --- /dev/null +++ b/rust/perspective-python/build/main.rs @@ -0,0 +1,81 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +use std::error::Error; +use std::fs; +use std::path::PathBuf; + +mod psp; + +fn main() -> Result<(), Box> { + pyo3_build_config::add_extension_module_link_args(); + // Pass the `perspective_client` links metadata flag so that the `inherit_docs` + // macro can find the path to the docs. + println!( + "cargo:rustc-env=PERSPECTIVE_CLIENT_DOCS_PATH={}", + std::env::var("DEP_PERSPECTIVE_CLIENT_DOCS_PATH").unwrap() + ); + + let manifest_dir = PathBuf::from(std::env::var("CARGO_MANIFEST_DIR")?); + if std::env::var("CARGO_FEATURE_EXTERNAL_CPP").is_ok() { + println!("cargo:warning=MESSAGE Building in development mode"); + for entry in fs::read_dir(manifest_dir.join("perspective"))? { + let entry = entry?; + if entry.file_name().to_string_lossy().ends_with("libpsp.so") { + fs::remove_file(entry.path())?; + } + } + } + + std::env::set_var("CARGO_FEATURE_PYTHON", "1"); // Not a relevant feature for this crate, but required for consistency + if let Some(artifact_dir) = psp::cmake_build()? { + let source_name = match std::env::var("CARGO_CFG_TARGET_OS")? + .to_lowercase() + .as_str() + { + "emscripten" => "psppy.wasm".to_string(), + "windows" => "libpsp.dll".to_string(), + _ => "libpsp.so".to_string(), + }; + + let ext = if cfg!(target_os = "windows") { + "dll" + } else { + "so" + }; + + let system = match std::env::var("CARGO_CFG_TARGET_OS")?.as_str() { + "macos" => "darwin".to_string(), + other => other.to_string(), + }; + + let machine = match std::env::var("CARGO_CFG_TARGET_ARCH")?.as_str() { + "aarch64" => "arm64".to_string(), + other => other.to_string(), + }; + + let dylib_name = format!("{}-{}-libpsp.{}", system, machine, ext); + let libpath = match std::env::var("CARGO_CFG_TARGET_OS")?.as_str() { + "windows" => artifact_dir + .join("build") + .join("MinSizeRel") + .join(&source_name), + _ => artifact_dir.join("build").join(&source_name), + }; + + let out = manifest_dir.join("perspective").join(dylib_name); + println!("cargo:rerun-if-changed={}", out.to_string_lossy()); + std::fs::copy(libpath, out).expect("Could not copy dylib to perspective/"); + } + + Ok(()) +} diff --git a/rust/perspective-python/build/psp.rs b/rust/perspective-python/build/psp.rs new file mode 100644 index 0000000000..37711cbf81 --- /dev/null +++ b/rust/perspective-python/build/psp.rs @@ -0,0 +1,190 @@ +// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +// ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +// ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +// ┃ Copyright (c) 2017, the Perspective Authors. ┃ +// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +// ┃ This file is part of the Perspective library, distributed under the terms ┃ +// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +// DO NOT EDIT THIS WITHOUT UPDATING rust/perspective-server/build/psp.rs + +use std::collections::HashSet; +use std::path::{Path, PathBuf}; +use std::{fs, io}; + +use cmake::Config; + +pub fn copy_dir_all( + src: impl AsRef, + dst: impl AsRef, + skip: &HashSet<&str>, +) -> io::Result<()> { + fs::create_dir_all(&dst)?; + for entry in fs::read_dir(src)? { + let entry = entry?; + let ty = entry.file_type()?; + if ty.is_dir() { + if !skip.contains(&*entry.file_name().to_string_lossy()) { + copy_dir_all(entry.path(), dst.as_ref().join(entry.file_name()), skip)?; + } + } else { + fs::copy(entry.path(), dst.as_ref().join(entry.file_name()))?; + } + } + + Ok(()) +} + +pub fn cmake_build() -> Result, std::io::Error> { + if std::env::var("CARGO_FEATURE_EXTERNAL_CPP").is_ok() { + println!("cargo:warning=MESSAGE Building in development mode"); + let root_dir_env = std::env::var("PSP_ROOT_DIR").expect("Must set PSP_ROOT_DIR"); + let root_dir = Path::new(root_dir_env.as_str()); + copy_dir_all(Path::join(root_dir, "cpp"), "cpp", &HashSet::from(["dist"]))?; + copy_dir_all(Path::join(root_dir, "cmake"), "cmake", &HashSet::new())?; + + println!( + "cargo:rerun-if-changed={}/cpp/perspective", + root_dir.display() + ); + } + + if matches!(std::env::var("DOCS_RS").as_deref(), Ok("1")) { + return Ok(None); + } + + let mut dst = Config::new("cpp/perspective"); + if cfg!(windows) && std::option_env!("CI").is_some() { + std::fs::create_dir_all("D:\\psp-build")?; + dst.out_dir("D:\\psp-build"); + } + + let profile = std::env::var("PROFILE").unwrap(); + dst.always_configure(true); + dst.define("CMAKE_BUILD_TYPE", profile.as_str()); + if std::env::var("PSP_ARCH").as_deref() == Ok("x86_64") { + dst.define("CMAKE_OSX_ARCHITECTURES", "x86_64"); + } else if std::env::var("PSP_ARCH").as_deref() == Ok("arm64") { + dst.define("CMAKE_OSX_ARCHITECTURES", "arm64"); + } + + if std::env::var("TARGET") + .unwrap_or_default() + .contains("wasm32") + { + dst.define("PSP_WASM_BUILD", "1"); + } else { + dst.define("PSP_WASM_BUILD", "0"); + } + + if cfg!(windows) { + dst.define( + "CMAKE_TOOLCHAIN_FILE", + format!( + "{}/scripts/buildsystems/vcpkg.cmake", + std::env::var("VCPKG_ROOT").unwrap().replace("\\", "/") + ), + ); + } + + if std::env::var("CARGO_FEATURE_PYTHON").is_ok() { + dst.define("CMAKE_POSITION_INDEPENDENT_CODE", "ON"); + dst.define("PSP_PYTHON_BUILD", "1"); + } + + if std::env::var("CARGO_FEATURE_EXTERNAL_CPP").is_err() { + dst.env("PSP_DISABLE_CLANGD", "1"); + } + + // WASM Exceptions don't work with the prebuilt Pyodide distribution. + // It must be rebuilt with WASM exceptions enabled + if std::env::var("CARGO_FEATURE_WASM_EXCEPTIONS").is_ok() { + dst.define("PSP_WASM_EXCEPTIONS", "1"); + } else { + dst.define("PSP_WASM_EXCEPTIONS", "0"); + } + + if !cfg!(windows) { + dst.build_arg(format!("-j{}", num_cpus::get())); + } + + println!("cargo:warning=MESSAGE Building cmake {}", profile); + let artifact_dir = dst.build(); + + Ok(Some(artifact_dir)) +} + +#[allow(dead_code)] +pub fn cmake_link_deps(cmake_build_dir: &Path) -> Result<(), std::io::Error> { + println!( + "cargo:rustc-link-search=native={}/build", + cmake_build_dir.display() + ); + + println!("cargo:rustc-link-lib=static=psp"); + link_cmake_static_archives(cmake_build_dir)?; + println!("cargo:rerun-if-changed=cpp/perspective"); + Ok(()) +} + +#[allow(dead_code)] +pub fn cxx_bridge_build() { + println!("cargo:warning=MESSAGE Building cxx"); + let mut compiler = cxx_build::bridge("src/ffi.rs"); + compiler + .file("src/server.cpp") + .include("include") + .include("cpp/perspective/src/include") + .std("c++17"); + // .flag("-fexceptions") // TODO not needed? + + if cfg!(windows) { + compiler.flag_if_supported("/c"); + } else { + compiler.static_flag(true); + } + + compiler.compile("perspective"); + + println!("cargo:rerun-if-changed=include/server.h"); + println!("cargo:rerun-if-changed=src/server.cpp"); + println!("cargo:rerun-if-changed=src/lib.rs"); +} + +/// Walk the cmake output path and emit link instructions for all archives. +/// TODO Can this be faster pls? +#[allow(dead_code)] +pub fn link_cmake_static_archives(dir: &Path) -> Result<(), std::io::Error> { + if dir.is_dir() { + for entry in fs::read_dir(dir)? { + let path = entry?.path(); + if path.is_dir() { + link_cmake_static_archives(&path)?; + } else { + let ext = path.extension().as_ref().map(|x| x.to_string_lossy()); + let stem = path.file_stem().as_ref().map(|x| x.to_string_lossy()); + let is_archive = (cfg!(windows) + && ext.as_deref() == Some("lib") + && stem.as_deref() != Some("perspective")) + || (!cfg!(windows) && ext.as_deref() == Some("a")); + if is_archive { + let a = if cfg!(windows) { + stem.unwrap().to_string() + } else { + stem.expect("bad")[3..].to_string() + }; + + // println!("cargo:warning=MESSAGE static link {}", a); + println!("cargo:rustc-link-search=native={}", dir.display()); + println!("cargo:rustc-link-lib=static={}", a); + } + } + } + } + + Ok(()) +} diff --git a/rust/perspective-python/package.json b/rust/perspective-python/package.json index ca23c81671..16b99bedda 100644 --- a/rust/perspective-python/package.json +++ b/rust/perspective-python/package.json @@ -7,6 +7,7 @@ "type": "git", "url": "https://github.com/finos/perspective" }, + "type": "module", "license": "Apache-2.0", "scripts": { "build": "node build.mjs", diff --git a/rust/perspective-python/perspective/__init__.py b/rust/perspective-python/perspective/__init__.py index 395d45d62e..24fd137406 100644 --- a/rust/perspective-python/perspective/__init__.py +++ b/rust/perspective-python/perspective/__init__.py @@ -25,7 +25,6 @@ from .perspective import ( PySyncClient, PerspectiveError, - PySyncServer, Table, View, PySyncProxySession as ProxySession, @@ -34,6 +33,7 @@ from .widget import PerspectiveWidget from .viewer import PerspectiveViewer +from .psp_cffi import ServerBase try: from .handlers import PerspectiveTornadoHandler @@ -45,7 +45,7 @@ def default_loop_cb(fn, *args, **kwargs): return fn(*args, **kwargs) -class Server(PySyncServer): +class Server(ServerBase): def set_threadpool_size(self, n_cpus): pass diff --git a/rust/perspective-python/perspective/psp_cffi.py b/rust/perspective-python/perspective/psp_cffi.py new file mode 100644 index 0000000000..851c82072e --- /dev/null +++ b/rust/perspective-python/perspective/psp_cffi.py @@ -0,0 +1,127 @@ +# ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓ +# ┃ ██████ ██████ ██████ █ █ █ █ █ █▄ ▀███ █ ┃ +# ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█ ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄ ▀█ █ ▀▀▀▀▀ ┃ +# ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄ █ ▄▄▄▄▄ ┃ +# ┃ █ ██████ █ ▀█▄ █ ██████ █ ███▌▐███ ███████▄ █ ┃ +# ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫ +# ┃ Copyright (c) 2017, the Perspective Authors. ┃ +# ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃ +# ┃ This file is part of the Perspective library, distributed under the terms ┃ +# ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ +# ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ + +import ctypes +import os +import platform +from typing import Callable + +if "PSP_CXX_SO_PATH" in os.environ: + lib = ctypes.CDLL(os.environ["PSP_CXX_SO_PATH"]) +else: + ext = "so" + if platform.system().lower() == "windows": + ext = "dll" + arch = platform.machine().lower() + if arch == "amd64": + arch = "x86_64" + dylib_name = f"{platform.system().lower()}-{arch}-libpsp.{ext}" + lib = ctypes.CDLL(os.path.join(os.path.dirname(__file__), dylib_name)) + + +class EncodedApiResp(ctypes.Structure): + _pack_ = 1 + _fields_ = [ + ("data", ctypes.POINTER(ctypes.c_char)), + ("size", ctypes.c_uint32), + ("client_id", ctypes.c_uint32), + ] + + +class EncodedApiEntries(ctypes.Structure): + _pack_ = 1 + _fields_ = [ + ("size", ctypes.c_uint32), + ("entries", ctypes.POINTER(EncodedApiResp)), + ] + + +ProtoApiServerPtr = ctypes.c_void_p +lib.psp_new_server.restype = ProtoApiServerPtr + +lib.psp_new_session.argtypes = [ProtoApiServerPtr] +lib.psp_new_session.restype = ctypes.c_uint32 + +lib.psp_handle_request.argtypes = [ + ProtoApiServerPtr, + ctypes.c_uint32, + ctypes.c_char_p, + ctypes.c_size_t, +] +lib.psp_handle_request.restype = ctypes.POINTER(EncodedApiEntries) + +lib.psp_poll.argtypes = [ProtoApiServerPtr] +lib.psp_poll.restype = ctypes.POINTER(EncodedApiEntries) + +lib.psp_new_session.argtypes = [ProtoApiServerPtr] +lib.psp_new_session.restype = ctypes.c_uint32 + +lib.psp_close_session.argtypes = [ProtoApiServerPtr, ctypes.c_uint32] +lib.psp_close_session.restype = None + +lib.psp_delete_server.argtypes = [ProtoApiServerPtr] +lib.psp_delete_server.restype = None + + +class Session: + def __init__(self, server: "ServerBase"): + self._server: "ServerBase" = server + self._session_id = lib.psp_new_session(server._server) + + def __del__(self): + lib.psp_close_session(self._server._server, self._session_id) + + def handle_request(self, bytes_msg): + resps = lib.psp_handle_request( + self._server._server, self._session_id, bytes_msg, len(bytes_msg) + ) + try: + size = resps.contents.size + for i in range(size): + resp = resps.contents.entries[i] + self._server._client_cbs[resp.client_id](resp.data[: resp.size]) + finally: + for i in range(size): + lib.psp_free(resps.contents.entries[i].data) + lib.psp_free(resps.contents.entries) + lib.psp_free(resps) + + def poll(self): + resps = lib.psp_poll(self._server._server) + try: + size = resps.contents.size + for i in range(size): + resp = resps.contents.entries[i] + self._server._client_cbs[resp.client_id](resp.data[: resp.size]) + finally: + for i in range(size): + lib.psp_free(resps.contents.entries[i].data) + lib.psp_free(resps.contents.entries) + lib.psp_free(resps) + + +class ServerBase: + def __init__(self): + self._client_cbs: dict[str, Callable[[bytes], None]] = {} + self._sessions: list[Session] = [] + self._server: int = lib.psp_new_server() + + def __del__(self): + for session in self._sessions: + del session + lib.psp_delete_server(self._server) + + def new_session(self, cb: Callable[[bytes], None]): + session = Session(self) + self._client_cbs[session._session_id] = cb + self._sessions.append(session) + return session diff --git a/rust/perspective-python/pyproject.toml b/rust/perspective-python/pyproject.toml index 52f15876a1..2b523ba95c 100644 --- a/rust/perspective-python/pyproject.toml +++ b/rust/perspective-python/pyproject.toml @@ -27,6 +27,10 @@ classifiers = [ module-name = "perspective" data = "perspective_python-3.0.0rc2.data" features = ["pyo3/extension-module"] +include = [ + { path = "perspective/*libpsp.so", format = "wheel" }, + { path = "perspective/*libpsp.dll", format = "wheel" }, +] [tool.pytest.ini_options] python_classes = ["Test", "Describe"] diff --git a/rust/perspective-python/src/lib.rs b/rust/perspective-python/src/lib.rs index 7f58470fbd..263e87b4e4 100644 --- a/rust/perspective-python/src/lib.rs +++ b/rust/perspective-python/src/lib.rs @@ -13,7 +13,7 @@ #![warn(unstable_features)] mod client; -mod server; +// mod server; pub use client::client_sync::{PySyncClient, PySyncProxySession, PySyncTable, PySyncView}; use pyo3::prelude::*; @@ -49,8 +49,8 @@ fn init_tracing() { fn perspective(py: Python, m: &Bound) -> PyResult<()> { init_tracing(); m.add_class::()?; - m.add_class::()?; - m.add_class::()?; + // m.add_class::()?; + // m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/rust/perspective-server/Cargo.toml b/rust/perspective-server/Cargo.toml index 1ba7507512..84999f05da 100644 --- a/rust/perspective-server/Cargo.toml +++ b/rust/perspective-server/Cargo.toml @@ -20,7 +20,7 @@ repository = "https://github.com/finos/perspective" license = "Apache-2.0" homepage = "https://perspective.finos.org" keywords = [] -build = "build.rs" +build = "build/main.rs" include = [ "src/**/*", "include/**/*", @@ -35,16 +35,17 @@ default = ["python"] external-cpp = [] wasm-exceptions = [] python = [] +export-cpp-only = [] [build-dependencies] -cxx-build = "1.0.115" +cxx-build = "1.0.92" cmake = "0.1.50" -num_cpus = "1.16.0" +num_cpus = "^1.15.0" [dependencies] perspective-client = { version = "3.0.0-rc.2", path = "../perspective-client" } async-lock = "2.5.0" -cxx = { version = "1.0.115", features = ["c++17"] } +cxx = { version = "1.0.92", features = ["c++17"] } tracing = { version = ">=0.1.36" } futures = "0.3" diff --git a/rust/perspective-python/build.rs b/rust/perspective-server/build/main.rs similarity index 84% rename from rust/perspective-python/build.rs rename to rust/perspective-server/build/main.rs index 35d04c4167..502cd38ae5 100644 --- a/rust/perspective-python/build.rs +++ b/rust/perspective-server/build/main.rs @@ -10,13 +10,13 @@ // ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃ // ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ -fn main() { - pyo3_build_config::add_extension_module_link_args(); +mod psp; - // Pass the `perspective_client` links metadata flag so that the `inherit_docs` - // macro can find the path to the docs. - println!( - "cargo:rustc-env=PERSPECTIVE_CLIENT_DOCS_PATH={}", - std::env::var("DEP_PERSPECTIVE_CLIENT_DOCS_PATH").unwrap() - ); +fn main() -> Result<(), std::io::Error> { + if let Some(artifact_dir) = psp::cmake_build()? { + psp::cmake_link_deps(&artifact_dir)?; + psp::cxx_bridge_build(); + } + + Ok(()) } diff --git a/rust/perspective-server/build.rs b/rust/perspective-server/build/psp.rs similarity index 90% rename from rust/perspective-server/build.rs rename to rust/perspective-server/build/psp.rs index 03b67e1814..fc2b7dede7 100644 --- a/rust/perspective-server/build.rs +++ b/rust/perspective-server/build/psp.rs @@ -11,12 +11,12 @@ // ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛ use std::collections::HashSet; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::{fs, io}; use cmake::Config; -fn copy_dir_all( +pub fn copy_dir_all( src: impl AsRef, dst: impl AsRef, skip: &HashSet<&str>, @@ -37,13 +37,14 @@ fn copy_dir_all( Ok(()) } -fn cmake_build() -> Result<(), std::io::Error> { +pub fn cmake_build() -> Result, std::io::Error> { if std::env::var("CARGO_FEATURE_EXTERNAL_CPP").is_ok() { println!("cargo:warning=MESSAGE Building in development mode"); let root_dir_env = std::env::var("PSP_ROOT_DIR").expect("Must set PSP_ROOT_DIR"); let root_dir = Path::new(root_dir_env.as_str()); copy_dir_all(Path::join(root_dir, "cpp"), "cpp", &HashSet::from(["dist"]))?; copy_dir_all(Path::join(root_dir, "cmake"), "cmake", &HashSet::new())?; + println!( "cargo:rerun-if-changed={}/cpp/perspective", root_dir.display() @@ -51,10 +52,15 @@ fn cmake_build() -> Result<(), std::io::Error> { } if matches!(std::env::var("DOCS_RS").as_deref(), Ok("1")) { - return Ok(()); + return Ok(None); } let mut dst = Config::new("cpp/perspective"); + if cfg!(windows) && std::option_env!("CI").is_some() { + std::fs::create_dir_all("D:\\psp-build")?; + dst.out_dir("D:\\psp-build"); + } + let profile = std::env::var("PROFILE").unwrap(); dst.always_configure(true); dst.define("CMAKE_BUILD_TYPE", profile.as_str()); @@ -105,7 +111,26 @@ fn cmake_build() -> Result<(), std::io::Error> { } println!("cargo:warning=MESSAGE Building cmake {}", profile); - let artifact = dst.build(); + let artifact_dir = dst.build(); + + Ok(Some(artifact_dir)) +} + +#[allow(dead_code)] +pub fn cmake_link_deps(cmake_build_dir: &Path) -> Result<(), std::io::Error> { + println!( + "cargo:rustc-link-search=native={}/build", + cmake_build_dir.display() + ); + + println!("cargo:rustc-link-lib=static=psp"); + link_cmake_static_archives(cmake_build_dir)?; + println!("cargo:rerun-if-changed=cpp/perspective"); + Ok(()) +} + +#[allow(dead_code)] +pub fn cxx_bridge_build() { println!("cargo:warning=MESSAGE Building cxx"); let mut compiler = cxx_build::bridge("src/ffi.rs"); compiler @@ -123,22 +148,15 @@ fn cmake_build() -> Result<(), std::io::Error> { compiler.compile("perspective"); - println!( - "cargo:rustc-link-search=native={}/build", - artifact.display() - ); - - link_cmake_static_archives(artifact.as_path())?; - println!("cargo:rerun-if-changed=cpp/perspective"); println!("cargo:rerun-if-changed=include/server.h"); println!("cargo:rerun-if-changed=src/server.cpp"); println!("cargo:rerun-if-changed=src/lib.rs"); - Ok(()) } /// Walk the cmake output path and emit link instructions for all archives. /// TODO Can this be faster pls? -fn link_cmake_static_archives(dir: &Path) -> Result<(), std::io::Error> { +#[allow(dead_code)] +pub fn link_cmake_static_archives(dir: &Path) -> Result<(), std::io::Error> { if dir.is_dir() { for entry in fs::read_dir(dir)? { let path = entry?.path(); @@ -168,7 +186,3 @@ fn link_cmake_static_archives(dir: &Path) -> Result<(), std::io::Error> { Ok(()) } - -fn main() -> Result<(), std::io::Error> { - cmake_build() -} diff --git a/rust/perspective-viewer/Cargo.toml b/rust/perspective-viewer/Cargo.toml index b6c527678e..59c9607bc3 100644 --- a/rust/perspective-viewer/Cargo.toml +++ b/rust/perspective-viewer/Cargo.toml @@ -40,6 +40,7 @@ crate-type = ["cdylib", "rlib"] path = "src/rust/lib.rs" [features] +external-bootstrap = [] metadata = [] default = [] @@ -117,7 +118,7 @@ tracing-subscriber = "0.3.15" wasm-bindgen = { version = "=0.2.92", features = ["serde-serialize"] } # Browser `Promise` bindings -wasm-bindgen-futures = "0.4.42" +wasm-bindgen-futures = "0.4.41" # Web framework yew = { version = "0.21.0", features = ["csr"] } diff --git a/rust/perspective-viewer/src/rust/lib.rs b/rust/perspective-viewer/src/rust/lib.rs index eea2d040cd..dd81283465 100644 --- a/rust/perspective-viewer/src/rust/lib.rs +++ b/rust/perspective-viewer/src/rust/lib.rs @@ -31,7 +31,9 @@ mod custom_events; mod dragdrop; pub mod exprtk; mod js; -mod model; + +#[doc(hidden)] +pub mod model; mod presentation; mod renderer; mod session; @@ -122,6 +124,7 @@ pub fn registerPlugin(name: &str) { /// Elements from JavaScript, as the methods themselves won't be defined yet. /// By default, this crate does not register `PerspectiveViewerElement` (as to /// preserve backwards-compatible synchronous API). +#[cfg(not(feature = "external-bootstrap"))] #[wasm_bindgen(js_name = "init")] pub fn js_init() { perspective_js::utils::set_global_logging(); @@ -136,6 +139,7 @@ pub fn js_init() { pub fn bootstrap_web_components(psp: &JsValue) { define_web_component::(psp); define_web_component::(psp); + define_web_component::(psp); define_web_component::(psp); } diff --git a/tools/perspective-scripts/sh.mjs b/tools/perspective-scripts/sh.mjs index ffb7fc0dd7..e55e2731e0 100644 --- a/tools/perspective-scripts/sh.mjs +++ b/tools/perspective-scripts/sh.mjs @@ -250,6 +250,7 @@ class Command extends Function { * shell scripts. `sh` knows how to remove consecutive text from fragments when * arguments are "falsey", which makes mapping flags to JS expressions a breeze. * + * @type {Command & {(): Command & {(): Promise}}} * @param {string} expression a bash command to be templated. * @returns {string} A command with the missing argument's flags removed. * @example