diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 195b0db..84e333a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -123,6 +123,40 @@ jobs: working-directory: build/test-suite run: ctest -C Release -V + build-on-ubuntu-for-wasm: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: asdf-vm/actions/install@v1 + - name: Install dependencies + uses: py-actions/py-dependency-install@v3 + with: + path: "requirements.txt" + - name: Install node + uses: actions/setup-node@v3 + with: + node-version: 20.9.0 + - name: Install browserify + run: npm install -g browserify + - name: Setup EMSDK + uses: mymindstorm/setup-emsdk@v11 + with: + version: 3.1.44 + - name: Install chrome + uses: browser-actions/setup-chrome@latest + - name: Use local djinni on branch for now + if: ${{ github.ref != 'refs/heads/master' }} + run: export CMAKE_DJINNI_OPT="-DDJINNI_EXECUTABLE=$(pwd)/bin/djinni" + - name: Report cmake version + run: cmake --version + - name: Configure cmake + run: cmake -DCMAKE_TOOLCHAIN_FILE=$EMSDK/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake -S . -B build -DDJINNI_WITH_WASM=ON -DDJINNI_EXECUTABLE=$(pwd)/bin/djinni + - name: Build release + run: cmake --build build --parallel $(nproc) --config Release + - name: Run headless test + run: pushd build && ctest --output-on-failure + - name: Run integration test on sample app + run: pushd build/examples && ./run.sh && python3 run-tests-selenium.py build-on-windows-for-cppcli: runs-on: windows-latest diff --git a/CMakeLists.txt b/CMakeLists.txt index f6691c9..ccb8545 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,6 +10,8 @@ project(djinni_support_lib CXX ${PROJECT_LANGUAGES}) include(GNUInstallDirs) +include(cmake/Djinni.cmake) + set(SRC_SHARED "djinni/djinni_common.hpp" "djinni/proxy_cache_interface.hpp" @@ -48,6 +50,11 @@ set(SRC_CPPCLI "djinni/cppcli/WrapperCache.cpp" ) +set(SRC_WASM + "djinni/wasm/djinni_wasm.hpp" + "djinni/wasm/djinni_wasm.cpp" +) + # set `DJINNI_LIBRARY_TYPE` to `STATIC` or `SHARED` to define the type of library. # If undefined, the type will be determined based on `BUILD_SHARED_LIBS` add_library(djinni_support_lib ${DJINNI_LIBRARY_TYPE} ${SRC_SHARED}) @@ -151,13 +158,14 @@ if(DJINNI_WITH_PYTHON) endif() option(DJINNI_WITH_CPPCLI "Include the C++/CLI support code in Djinni support library." OFF) +option(DJINNI_WITH_WASM "Include the WASM/TS support code in Djinni support library." OFF) if(DJINNI_WITH_CPPCLI) if(NOT MSVC) message(FATAL_ERROR "Enabling DJINNI_WITH_CPPCLI without MSVC is not supported") endif() - if(DJINNI_WITH_OBJC OR DJINNI_WITH_JNI OR DJINNI_WITH_PYTHON) + if(DJINNI_WITH_OBJC OR DJINNI_WITH_JNI OR DJINNI_WITH_PYTHON OR DJINNI_WITH_WASM) message(FATAL_ERROR "DJINNI_WITH_CPPCLI can not be used with other bindings enabled.") endif() @@ -180,13 +188,31 @@ if(DJINNI_WITH_CPPCLI) ) endif() -if(NOT (DJINNI_WITH_OBJC OR DJINNI_WITH_JNI OR DJINNI_WITH_PYTHON OR DJINNI_WITH_CPPCLI)) - message(FATAL_ERROR "At least one of DJINNI_WITH_OBJC or DJINNI_WITH_JNI or DJINNI_WITH_PYTHON or DJINNI_WITH_CPPCLI must be enabled.") +if(DJINNI_WITH_WASM) + + if(DJINNI_WITH_OBJC OR DJINNI_WITH_JNI OR DJINNI_WITH_PYTHON OR DJINNI_WITH_CPPCLI) + message(FATAL_ERROR "DJINNI_WITH_WASM can not be used with other bindings enabled.") + endif() + + target_include_directories(djinni_support_lib PUBLIC "$") + target_sources(djinni_support_lib PRIVATE ${SRC_WASM}) + source_group("wasm" FILES ${SRC_WASM}) + install( + FILES + "djinni/wasm/djinni_wasm.hpp" + DESTINATION + ${CMAKE_INSTALL_INCLUDEDIR}/djinni/wasm + ) +endif() + +if(NOT (DJINNI_WITH_OBJC OR DJINNI_WITH_JNI OR DJINNI_WITH_PYTHON OR DJINNI_WITH_CPPCLI OR DJINNI_WITH_WASM)) + message(FATAL_ERROR "At least one of DJINNI_WITH_OBJC or DJINNI_WITH_JNI or DJINNI_WITH_PYTHON or DJINNI_WITH_CPPCLI or DJINNI_WITH_WASM must be enabled.") endif() option(DJINNI_BUILD_TESTING "Build tests" ON) include(CTest) if (BUILD_TESTING AND DJINNI_BUILD_TESTING) + add_subdirectory(examples) add_subdirectory(test-suite) endif() diff --git a/bin/djinni b/bin/djinni new file mode 100755 index 0000000..9629bab Binary files /dev/null and b/bin/djinni differ diff --git a/test-suite/Djinni.cmake b/cmake/Djinni.cmake similarity index 91% rename from test-suite/Djinni.cmake rename to cmake/Djinni.cmake index 19a188f..577f6a0 100644 --- a/test-suite/Djinni.cmake +++ b/cmake/Djinni.cmake @@ -15,7 +15,6 @@ execute_process(COMMAND ${DJINNI_EXECUTABLE} "--version" OUTPUT_VARIABLE DJINNI_ string(REGEX REPLACE "\n+$" "" DJINNI_VERSION "${DJINNI_VERSION}") message(STATUS "Found Djinni: ${DJINNI_EXECUTABLE} (${DJINNI_VERSION})") - macro(append_if_defined LIST OPTION) if(NOT "${ARGN}" STREQUAL "") list(APPEND ${LIST} ${OPTION} ${ARGN}) @@ -178,6 +177,15 @@ function(add_djinni_target) CPPCLI_NAMESPACE CPPCLI_INCLUDE_CPP_PREFIX + WASM_OUT + WASM_OUT_FILES + WASM_NAMESPACE + + TS_OUT + TS_OUT_FILES + TS_MODULE + TS_SUPPORT_FILES_OUT + YAML_OUT YAML_OUT_FILE YAML_PREFIX @@ -270,6 +278,10 @@ function(add_djinni_target) append_if_defined(DJINNI_GENERATION_COMMAND "--cppcli-namespace" ${DJINNI_CPPCLI_NAMESPACE}) append_if_defined(DJINNI_GENERATION_COMMAND "--cppcli-include-cpp-prefix" ${DJINNI_CPPCLI_INCLUDE_CPP_PREFIX}) + append_if_defined(DJINNI_GENERATION_COMMAND "--wasm-namespace" ${DJINNI_WASM_NAMESPACE}) + append_if_defined(DJINNI_GENERATION_COMMAND "--ts-module" ${DJINNI_TS_MODULE}) + append_if_defined(DJINNI_GENERATION_COMMAND "--ts-support-files-out" ${DJINNI_TS_SUPPORT_FILES_OUT}) + if(DEFINED DJINNI_CPP_OUT_FILES) set(DJINNI_CPP_GENERATION_COMMAND ${DJINNI_GENERATION_COMMAND}) append_if_defined(DJINNI_CPP_GENERATION_COMMAND "--cpp-out" ${DJINNI_CPP_OUT}) @@ -418,5 +430,38 @@ function(add_djinni_target) set(${DJINNI_CPPCLI_OUT_FILES} ${CPPCLI_OUT_FILES} PARENT_SCOPE) endif() + if(DEFINED DJINNI_WASM_OUT_FILES) + set(DJINNI_WASM_GENERATION_COMMAND ${DJINNI_GENERATION_COMMAND}) + append_if_defined(DJINNI_WASM_GENERATION_COMMAND "--wasm-out" ${DJINNI_WASM_OUT}) + + resolve_djinni_outputs(COMMAND "${DJINNI_WASM_GENERATION_COMMAND}" RESULT WASM_OUT_FILES) + + add_custom_command( + OUTPUT ${WASM_OUT_FILES} + DEPENDS ${DJINNI_INPUTS} + COMMAND ${DJINNI_WASM_GENERATION_COMMAND} + COMMENT "Generating Djinni Web Assembly bindings from ${DJINNI_IDL}" + VERBATIM + ) + set(${DJINNI_WASM_OUT_FILES} ${WASM_OUT_FILES} PARENT_SCOPE) + endif() + + if(DEFINED DJINNI_TS_OUT_FILES) + set(DJINNI_TS_GENERATION_COMMAND ${DJINNI_GENERATION_COMMAND}) + append_if_defined(DJINNI_TS_GENERATION_COMMAND "--ts-out" ${DJINNI_TS_OUT}) + + resolve_djinni_outputs(COMMAND "${DJINNI_TS_GENERATION_COMMAND}" RESULT TS_OUT_FILES) + + message(STATUS "Output: ${TS_OUT_FILES}, Depends: ${DJINNI_INPUTS}, Command: ${DJINNI_TS_GENERATION_COMMAND}") + add_custom_command( + OUTPUT ${TS_OUT_FILES} + DEPENDS ${DJINNI_INPUTS} + COMMAND ${DJINNI_TS_GENERATION_COMMAND} + COMMENT "Generating Djinni TS bindings from ${DJINNI_IDL}" + VERBATIM + ) + set(${DJINNI_TS_OUT_FILES} ${TS_OUT_FILES} PARENT_SCOPE) + endif() + endfunction() diff --git a/djinni/wasm/djinni_wasm.cpp b/djinni/wasm/djinni_wasm.cpp new file mode 100644 index 0000000..0b05cb4 --- /dev/null +++ b/djinni/wasm/djinni_wasm.cpp @@ -0,0 +1,222 @@ +/** + * Copyright 2021 Snap, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "djinni_wasm.hpp" + +namespace djinni { + +Binary::CppType Binary::toCpp(const JsType& j) { + return PrimitiveArray, Binary>::toCpp(j); +} +Binary::JsType Binary::fromCpp(const CppType& c) { + static em::val arrayClass = em::val::global("Uint8Array"); + return PrimitiveArray, Binary>::fromCpp(c); +} +em::val Binary::getArrayClass() { + static em::val arrayClass = em::val::global("Uint8Array"); + return arrayClass; +} + +Date::CppType Date::toCpp(const JsType& j) { + auto nanosSinceEpoch = std::chrono::nanoseconds(static_cast(j.call("getTime").as() * 1'000'000)); + return CppType(std::chrono::duration_cast(nanosSinceEpoch)); +} +Date::JsType Date::fromCpp(const CppType& c) { + auto nanosSinceEpoch = std::chrono::duration_cast(c.time_since_epoch()); + static em::val dateType = em::val::global("Date"); + return dateType.new_(static_cast(nanosSinceEpoch.count()) / 1'000'000.0); +} + +JsProxyId nextId = 0; +std::unordered_map> jsProxyCache; +std::unordered_map cppProxyCache; +std::mutex jsProxyCacheMutex; +std::mutex cppProxyCacheMutex; + +JsProxyBase::JsProxyBase(const em::val& v) : _js(v), _id(_js["_djinni_js_proxy_id"].as()) { +} + +JsProxyBase::~JsProxyBase() { + std::lock_guard lk(jsProxyCacheMutex); + jsProxyCache.erase(_id); +} + +const em::val& JsProxyBase::_jsRef() const { + return _js; +} + +void JsProxyBase::checkError(const em::val& v) { + if (v.instanceof(em::val::global("Error"))) { + auto cppExceptionPtr = v["_djinni_cpp_exception_ptr"]; + if (!cppExceptionPtr.isUndefined()) { + std::exception_ptr* exptr = reinterpret_cast(cppExceptionPtr.as()); + std::rethrow_exception(*exptr); + } else { + throw JsException(v); + } + } +} + +void checkForNull(void* ptr, const char* context) { + if (!ptr) { + throw std::invalid_argument(std::string("nullptr is not allowed in ") + context); + } +} + +em::val getCppProxyFinalizerRegistry() { + static auto inst = em::val::module_property("cppProxyFinalizerRegistry"); + return inst; +} + +em::val getCppProxyClass() { + static auto inst = em::val::module_property("DjinniCppProxy"); + return inst; +} + +em::val getWasmMemoryBuffer() { + // When ALLOW_MEMORY_GROWTH is turned on, the WebAssembly.Memory object's underlying buffer changes as it grows, + // and the HEAP* views in the Emscripten module object get reset to new views over the bigger memory buffer. + // In this mode, capturing the heap here as a static variable is incorrect, and leads to runtime errors. + // https://github.com/emscripten-core/emscripten/blob/3.1.7/src/settings.js#L194-L207 is the growth setting + // https://github.com/emscripten-core/emscripten/blob/3.1.7/src/library.js#L127 is the call to reset the views after grow() + // https://github.com/emscripten-core/emscripten/blob/3.1.7/src/preamble.js#L269-L286 is where the views get reset + return em::val::module_property("HEAPU32")["buffer"]; +} + +em::val DataObject::createJsObject() { + static auto finalizerRegistry = em::val::module_property("directBufferFinalizerRegistry"); + static auto uint8ArrayClass = em::val::global("Uint8Array"); + em::val jsObj = uint8ArrayClass.new_(getWasmMemoryBuffer(), addr(), size()); + finalizerRegistry.call("register", jsObj, reinterpret_cast(this)); + return jsObj; +} + +static em::val allocateWasmBuffer(unsigned size) { + auto* dbuf = new GenericBuffer>(size); + return dbuf->createJsObject(); +} + +extern "C" EMSCRIPTEN_KEEPALIVE +void releaseWasmBuffer(unsigned addr) { + delete reinterpret_cast(addr); +} + +EM_JS(void, djinni_init_wasm, (), { + // console.log("djinni_init_wasm"); + Module.cppProxyFinalizerRegistry = new FinalizationRegistry(nativeRef => { + // console.log("finalizing cpp object @" + nativeRef); + nativeRef.nativeDestroy(); + nativeRef.delete(); + }); + + Module.directBufferFinalizerRegistry = new FinalizationRegistry(addr => { + Module._releaseWasmBuffer(addr); + }); + + class DjinniCppProxy { + constructor(nativeRef, methods) { + // console.log('new cpp proxy'); + this._djinni_native_ref = nativeRef; + let self = this; + methods.forEach(function(method) { + self[method] = function(...args) { + return nativeRef[method](...args); + } + }); + } + } + Module.DjinniCppProxy = DjinniCppProxy; + + class DjinniJsPromiseBuilder { + constructor(cppHandlerPtr) { + this.promise = new Promise((resolveFunc, rejectFunc) => { + Module.initCppResolveHandler(cppHandlerPtr, resolveFunc, rejectFunc); + }); + } + } + Module.DjinniJsPromiseBuilder = DjinniJsPromiseBuilder; + + Module.makeNativePromiseResolver = function(func, pNativePromise) { + return function(res) { + Module.resolveNativePromise(func, pNativePromise, res); + }; + }; + Module.makeNativePromiseRejecter = function(func, pNativePromise) { + return function(err) { + Module.rejectNativePromise(func, pNativePromise, err); + }; + }; + + Module.writeNativeMemory = function(src, nativePtr) { + var srcByteView = new Uint8Array(src.buffer, src.byteOffset, src.byteLength); + Module.HEAPU8.set(srcByteView, nativePtr); + }; + Module.readNativeMemory = function(cls, nativePtr, nativeSize) { + return new cls(Module.HEAPU8.buffer.slice(nativePtr, nativePtr + nativeSize)); + }; + + Module.protobuf = {}; + Module.registerProtobufLib = function(name, proto) { + Module.protobuf[name] = proto; + }; + + Module.callJsProxyMethod = function(obj, method, ...args) { + try { + return obj[method].apply(obj, args); + } catch (e) { + return e; + } + }; +}); + +EM_JS(void, djinni_register_name_in_ns, (const char* prefixedName, const char* namespacedName), { + prefixedName = readLatin1String(prefixedName); + namespacedName = readLatin1String(namespacedName); + let parts = namespacedName.split('.'); + let name = parts.pop(); + let ns = parts.reduce(function(path, part) { + if (!path.hasOwnProperty(part)) { path[part] = {}}; + return path[part] + }, Module); + ns[name] = Module[prefixedName]; +}); + +em::val djinni_native_exception_to_js(const std::exception& e) { + if (const auto* jsEx = dynamic_cast(&e)) { + return jsEx->cause(); + } else { + static std::exception_ptr exptr; + static auto ErrorClass = em::val::global("Error"); + auto error = ErrorClass.new_(std::string("C++: ") + e.what()); + exptr = std::current_exception(); + error.set("_djinni_cpp_exception_ptr", em::val(reinterpret_cast(&exptr))); + return error; + } +} + +void djinni_throw_native_exception(const std::exception& e) { + djinni_native_exception_to_js(e).throw_(); +} + +EMSCRIPTEN_BINDINGS(djinni_wasm) { + djinni_init_wasm(); + em::function("allocateWasmBuffer", &allocateWasmBuffer); + em::function("initCppResolveHandler", &CppResolveHandlerBase::initInstance); + em::function("resolveNativePromise", &CppResolveHandlerBase::resolveNativePromise); + em::function("rejectNativePromise", &CppResolveHandlerBase::rejectNativePromise); +} + +} diff --git a/djinni/wasm/djinni_wasm.hpp b/djinni/wasm/djinni_wasm.hpp new file mode 100644 index 0000000..8add002 --- /dev/null +++ b/djinni/wasm/djinni_wasm.hpp @@ -0,0 +1,657 @@ +/** + * Copyright 2021 Snap, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +#ifdef __EMSCRIPTEN_PTHREADS__ +#include +#endif + +#include +#include +#include +#include +#include +#include + +namespace em = emscripten; + +namespace djinni { + +extern em::val getCppProxyFinalizerRegistry(); +extern em::val getCppProxyClass(); +extern em::val getWasmMemoryBuffer(); + +template +class InstanceTracker { +#ifdef DJINNI_WASM_TRACK_INSTANCES + static int& count() { + static int theCount = 0; + return theCount; + } +public: + InstanceTracker() { + std::cout << "++" << typeid(this).name() << " => " << ++count() << std::endl; + } + virtual ~InstanceTracker() { + std::cout << "--" << typeid(this).name() << " => " << --count() << std::endl; + } +#endif +}; + +template +class Primitive { +public: + using CppType = T; + using JsType = T; + + struct Boxed { + using JsType = em::val; + static CppType toCpp(JsType j) { + return j.as(); + } + static JsType fromCpp(CppType c) { + return JsType(c); + } + }; + + static CppType toCpp(const JsType& j) { + return j; + } + static JsType fromCpp(const CppType& c) { + return c; + } +}; + +using Bool = Primitive; +using I8 = Primitive; +using I16 = Primitive; +using I32 = Primitive; +using I64 = Primitive; +using F32 = Primitive; +using F64 = Primitive; +using String = Primitive; +using WString = Primitive; + +template +class WasmEnum { +public: + using CppType = T; + using JsType = int32_t; + + struct Boxed { + using JsType = em::val; + static CppType toCpp(JsType j) { + return static_cast(j.as()); + } + static JsType fromCpp(CppType c) { + return JsType(static_cast(c)); + } + }; + + static CppType toCpp(const JsType& j) { + return static_cast(j); + } + static JsType fromCpp(const CppType& c) { + return static_cast(c); + } +}; + +class Binary { +public: + using CppType = std::vector; + using JsType = em::val; + using Boxed = Binary; + + static CppType toCpp(const JsType& j); + static JsType fromCpp(const CppType& c); + + static em::val getArrayClass(); +}; + +class Date { +public: + using CppType = std::chrono::system_clock::time_point; + using JsType = em::val; + using Boxed = Date; + + static CppType toCpp(const JsType& j); + static JsType fromCpp(const CppType& c); +}; + +template