Merge pull request #21491 from oleiman/xform/sdk/core-5093/sr-js

[CORE-5093] Data Transforms SDK: Schema Registry Support for JavaScript

src/transform-sdk/js/CMakeLists.txt

@@ -24,6 +24,20 @@ endif()
 include(FetchContent)
 set(FETCHCONTENT_QUIET FALSE)
 
+if(CMAKE_BUILD_TYPE MATCHES Release)
+  include(CheckIPOSupported)
+  check_ipo_supported(RESULT ltosupported OUTPUT error)
+  # NOTE(oren): rather a blunt instrument. CMake doesn't currently
+  # offer the ability to customize the LTO setting, and thin LTO is not
+  # sufficient for our use case, so we blanket enable full LTO for all
+  # targets. Predicated on compiler ID becuase these options are specific
+  # to clang.
+  if(ltosupported AND (CMAKE_CXX_COMPILER_ID MATCHES "Clang"))
+    add_compile_options(-flto=full)
+    add_link_options(-flto=full)
+  endif()
+endif()
+
 FetchContent_Declare(
   quickjs
   GIT_REPOSITORY https://github.com/quickjs-ng/quickjs.git
@@ -65,12 +79,10 @@ target_link_libraries(
   qjs Redpanda::transform_sdk Redpanda::js_vm
 )
 
-if(CMAKE_BUILD_TYPE MATCHES Release)
-  include(CheckIPOSupported)
-  check_ipo_supported(RESULT ltosupported OUTPUT error)
-  if(ltosupported)
-    set_property(TARGET redpanda_js_transform PROPERTY INTERPROCEDURAL_OPTIMIZATION ON)
-  endif()
+# NOTE(oren): Fall back to standard property if we decided LTO was needed 
+# and happen to be building non-clang for some reason.
+if (ltosupported AND NOT (CMAKE_CXX_COMPILER_ID MATCHES "Clang"))
+  set_property(TARGET redpanda_js_transform PROPERTY INTERPROCEDURAL_OPTIMIZATION ON)
 endif()
 
 # Tests

src/transform-sdk/js/js_vm.cc

@@ -19,6 +19,7 @@
 #include <sys/types.h>
 
 #include <cassert>
+#include <cmath>
 #include <cstdint>
 #include <cstdio>
 #include <expected>
@@ -126,18 +127,42 @@ value value::array_buffer(JSContext* ctx, std::span<uint8_t> data) {
         ctx, data.data(), data.size(), func, opaque, /*is_shared=*/0)};
 }
 
+std::expected<value, exception>
+value::array_buffer_copy(JSContext* ctx, std::span<uint8_t> data) {
+    value val{ctx, JS_NewArrayBufferCopy(ctx, data.data(), data.size())};
+    if (val.is_exception()) {
+        return std::unexpected(exception::current(ctx));
+    }
+    return val;
+}
+
 value value::uint8_array(JSContext* ctx, std::span<uint8_t> data) {
     void* opaque = nullptr;
     // NOLINTNEXTLINE(*easily-swappable-parameters)
     JSFreeArrayBufferDataFunc* func = [](JSRuntime*, void* opaque, void* ptr) {
         // Nothing to do
     };
+
     return {
       ctx,
       JS_NewUint8Array(
         ctx, data.data(), data.size(), func, opaque, /*is_shared=*/0)};
 }
 
+// see quickjs source for detailed buffer ownership semantics
+// https://github.com/quickjs-ng/quickjs/blob/da5b95dcaf372dcc206019e171a0b08983683bf5/quickjs.c#L49379-L49436
+// TL;DR - With copying enabled, the array constructor registers a baked-in
+// free_func to be called by the object destructor. The copy of data is made
+// and mangaged internally to the quickjs runtime.
+std::expected<value, exception>
+value::uint8_array_copy(JSContext* ctx, std::span<uint8_t> data) {
+    value val{ctx, JS_NewUint8ArrayCopy(ctx, data.data(), data.size())};
+    if (val.is_exception()) {
+        return std::unexpected(exception::current(ctx));
+    }
+    return val;
+}
+
 void value::detach_buffer() {
     assert(is_array_buffer());
     JS_DetachArrayBuffer(_ctx, _underlying);
@@ -182,6 +207,7 @@ value value::current_exception(JSContext* ctx) {
 }
 bool value::is_number() const { return JS_IsNumber(_underlying) != 0; }
 bool value::is_exception() const { return JS_IsException(_underlying) != 0; }
+bool value::is_error() const { return JS_IsError(_ctx, _underlying) != 0; }
 bool value::is_function() const {
     return JS_IsFunction(_ctx, _underlying) != 0;
 }
@@ -205,6 +231,11 @@ double value::as_number() const {
     return JS_VALUE_GET_INT(_underlying);
 }
 
+int32_t value::as_integer() const {
+    auto num = as_number();
+    return static_cast<int32_t>(std::lround(num));
+}
+
 std::expected<value, exception> value::call(std::span<value> values) {
     assert(is_function());
     std::vector<JSValue> raw;
@@ -275,6 +306,7 @@ size_t value::array_length() const {
     return JS_VALUE_GET_INT(prop.raw());
 }
 
+// NOLINTNEXTLINE(*-no-recursion)
 std::string value::debug_string() const {
     if (is_null()) {
         return "null";
@@ -284,12 +316,18 @@ std::string value::debug_string() const {
     }
     size_t size = 0;
     const char* str = JS_ToCStringLen(_ctx, &size, _underlying);
+    std::string result;
     if (str != nullptr) {
-        auto result = std::string(str, size);
+        result = std::string(str, size);
         JS_FreeCString(_ctx, str);
-        return result;
+    } else {
+        result = "[exception]";
+    }
+    if (is_exception() || is_error()) {
+        auto stack = get_property("stack");
+        result += std::format("\n Stack: \n{}", stack.debug_string());
     }
-    return "[exception]";
+    return result;
 }
 
 bool operator==(const value& lhs, const value& rhs) {
@@ -462,7 +500,11 @@ std::expected<std::monostate, exception> runtime::create_builtins() {
     if (!result.has_value()) {
         return result;
     }
-    return global_this.set_property("process", process);
+    result = global_this.set_property("process", process);
+    if (!result.has_value()) {
+        return result;
+    }
+    return global_this.set_property("self", value::global(_ctx.get()));
 }
 
 std::expected<compiled_bytecode, exception>

src/transform-sdk/js/js_vm.h

@@ -140,6 +140,10 @@ class value {
      */
     static value array_buffer(JSContext* ctx, std::span<uint8_t> data);
 
+    /** Create a typed array buffer value (copy) */
+    static std::expected<value, exception>
+    array_buffer_copy(JSContext* ctx, std::span<uint8_t> data);
+
     /**
      * Create an array buffer value (view - no copies).
      *
@@ -150,6 +154,10 @@ class value {
      */
     static value uint8_array(JSContext* ctx, std::span<uint8_t> data);
 
+    /** Create a typed uint8 array value (copy) */
+    static std::expected<value, exception>
+    uint8_array_copy(JSContext* ctx, std::span<uint8_t> data);
+
     /**
      * Free the underlying memory to an array buffer.
      */
@@ -196,6 +204,8 @@ class value {
     [[nodiscard]] bool is_number() const;
     /** Is this value an exception? */
     [[nodiscard]] bool is_exception() const;
+    /** Is this value an error? */
+    [[nodiscard]] bool is_error() const;
     /** Is this value a function? */
     [[nodiscard]] bool is_function() const;
     /** Is this value a string? */
@@ -216,6 +226,9 @@ class value {
     /** Get the number from the object. */
     [[nodiscard]] double as_number() const;
 
+    /** Get an integer from the object. Rounding. */
+    [[nodiscard]] int32_t as_integer() const;
+
     /**
      * Return a reference to the raw JSValue without incrementing the ref
      * count.