[GR-48942] Run C extensions natively

PullRequest: truffleruby/4018

Author: eregon

.gitignore

@@ -48,6 +48,8 @@ lib/gems/gems/debug-*/ext/debug/debug_version.h
 src/main/c/*/Makefile
 !src/main/c/spawn-helper/Makefile
 !src/main/c/truffleposix/Makefile
+src/main/c/cext-trampoline/trampoline.c
+src/main/c/cext/wrappers.c
 src/main/c/etc/constdefs.h
 src/main/c/spawn-helper/spawn-helper
 

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 New features:
 
+* C/C++ extensions are now compiled using the system toolchain and executed natively instead of using GraalVM LLVM (Sulong). This leads to faster startup, no warmup, better compatibility and faster installation for C/C++ extensions (#3118, @eregon).
 
 Bug fixes:
 
@@ -11,7 +12,6 @@ Bug fixes:
 
 Compatibility:
 
-* Fix problems with the LLVM toolchain wrappers with Xcode 15 on macOS.
 * Add `Exception#detailed_message` method (#3257, @andrykonchin).
 * Fix `rb_enc_vsprintf` and force String encoding instead of converting it (@andrykonchin).
 * Add `rb_gc_mark_movable` function (@andrykonchin).

ci.jsonnet

@@ -232,14 +232,18 @@ local part_definitions = {
   },
 
   platform: {
-    local common_deps = common.deps.truffleruby + common.deps.sulong,
-    local linux_amd64_extra_deps = {
-      packages+: {
-        binutils: ">=2.30",
-      },
-    },
-
-    linux: common.linux_amd64 + common_deps + linux_amd64_extra_deps + {
+    local devtoolset = { # Until there is a proper object in common.jsonnet for it
+      packages+: if self.os == "linux" then
+        (if self.arch == "aarch64" then {
+          "00:devtoolset": "==10",
+        } else {
+          "00:devtoolset": "==11",
+        })
+      else {},
+    },
+    local common_deps = common.deps.truffleruby + common.deps.sulong + devtoolset,
+
+    linux: common.linux_amd64 + common_deps + {
       platform_name:: "LinuxAMD64",
       "$.cap":: {
         normal_machine: [],
@@ -308,6 +312,7 @@ local part_definitions = {
       run+: jt(["test", "specs", ":all"]) +
             jt(["test", "specs", ":tracepoint"]) +
             jt(["test", "specs", ":next"]) +
+            [["env", "SPEC_CAPI_CXX=true"] + jt(["test", "specs", "--timeout", "180", ":capi"])[0]] +
             jt(["test", "basictest"]),
     },
 
@@ -371,6 +376,13 @@ local part_definitions = {
             jt(["test", "bundle"]),
     },
 
+    test_cexts_sulong: {
+      environment+: {
+        TRUFFLERUBYOPT: "--experimental-options --cexts-sulong",
+      },
+      run+: jt(["test", "specs", ":cext"]),
+    },
+
     testdownstream_aot: { run+: [["mx", "ruby_testdownstream_aot", "$RUBY_BIN"]] },
 
     test_make_standalone_distribution: {
@@ -381,18 +393,12 @@ local part_definitions = {
 
     generate_native_config: {
       setup+: [
-        ["mx", "sforceimports"], # clone the graal repo
-        ["mx", "-p", "../graal/sulong", "build"],
-        ["set-export", "TOOLCHAIN_PATH", ["mx", "-p", "../graal/sulong", "lli", "--print-toolchain-path"]],
+        ["ruby", "-e", "pp RbConfig::MAKEFILE_CONFIG"], # For convenience
       ],
       run+: [
-        ["env",
-          "LD_LIBRARY_PATH=$BUILD_DIR/graal/sulong/mxbuild/" + self.os + "-" + self.arch + "/SULONG_HOME/native/lib:$LD_LIBRARY_PATH", # for finding libc++
-          "PATH=$TOOLCHAIN_PATH:$PATH",
-          "ruby", "tool/generate-native-config.rb"],
+        ["ruby", "tool/generate-native-config.rb"],
         ["cat", "src/main/java/org/truffleruby/platform/" + self.platform_name + "NativeConfiguration.java"],
 
-        # Uses the system compiler as using the toolchain for this does not work on macOS
         ["tool/generate-config-header.sh"],
         ["cat", "lib/cext/include/truffleruby/config_" + self.os + "_" + self.arch + ".h"],
       ],
@@ -537,6 +543,7 @@ local composition_environment = utils.add_inclusion_tracking(part_definitions, "
       "ruby-test-cexts-linux-aarch64":  $.platform.linux_aarch64 + $.jdk.stable + $.env.jvm + gate + $.use.gem_test_pack + $.use.sqlite331 + $.run.test_cexts,
       "ruby-test-cexts-darwin-amd64":   $.platform.darwin_amd64 + $.jdk.stable + $.env.jvm + gate + $.use.gem_test_pack + $.run.test_cexts + { timelimit: "01:30:00" },
       "ruby-test-cexts-darwin-aarch64": $.platform.darwin_aarch64 + $.jdk.stable + $.env.jvm + gate + $.use.gem_test_pack + $.run.test_cexts + { timelimit: "00:40:00" },
+      "ruby-test-cexts-sulong":         $.platform.linux  + $.jdk.stable + $.env.jvm + gate + $.run.test_cexts_sulong + { timelimit: "20:00" },
       "ruby-test-gems-linux-amd64":     $.platform.linux  + $.jdk.stable + $.env.jvm + gate + $.use.gem_test_pack + $.run.test_gems,
       "ruby-test-gems-darwin-amd64":    $.platform.darwin_amd64 + $.jdk.stable + $.env.jvm + gate + $.use.gem_test_pack + $.run.test_gems,
       "ruby-test-gems-darwin-aarch64":  $.platform.darwin_aarch64 + $.jdk.stable + $.env.jvm + gate + $.use.gem_test_pack + $.run.test_gems,
@@ -701,11 +708,12 @@ local composition_environment = utils.add_inclusion_tracking(part_definitions, "
 
   manual_builds: {
     local shared = $.use.common + $.cap.manual + { timelimit: "15:00" },
+    local native_config = $.run.generate_native_config + $.run.check_native_config,
 
-    "ruby-generate-native-config-linux-amd64":    $.platform.linux + $.jdk.stable + shared + $.run.generate_native_config,
-    "ruby-generate-native-config-linux-aarch64":  $.platform.linux_aarch64 + $.jdk.stable + shared + $.run.generate_native_config,
-    "ruby-generate-native-config-darwin-amd64":   $.platform.darwin_amd64 + $.jdk.stable + shared + $.run.generate_native_config,
-    "ruby-generate-native-config-darwin-aarch64": $.platform.darwin_aarch64 + $.jdk.stable + shared + $.run.generate_native_config,
+    "ruby-generate-native-config-linux-amd64":    $.platform.linux + $.jdk.stable + shared + native_config,
+    "ruby-generate-native-config-linux-aarch64":  $.platform.linux_aarch64 + $.jdk.stable + shared + native_config,
+    "ruby-generate-native-config-darwin-amd64":   $.platform.darwin_amd64 + $.jdk.stable + shared + native_config,
+    "ruby-generate-native-config-darwin-aarch64": $.platform.darwin_aarch64 + $.jdk.stable + shared + native_config,
   },
 
   builds:

doc/contributor/how-to-guide.md

@@ -1292,15 +1292,15 @@ When you modify C-files (*.c, *.h) or compilation-related Ruby files
 affect the ABI or increase the ABI *check* version explicitly by modifying one of
 the files:
 
-- lib/cext/ABI_version.txt
+- lib/cext/include/truffleruby/truffleruby-abi-version.h
 - lib/cext/ABI_check.txt
 
 ABI change is:
 - changing headers or compilation flags
 - removing/adding a non-static function
 - implementing already declared non-static functions
 
-In case of doubt, bump `ABI_version.txt`.
+In case of doubt, bump `truffleruby-abi-version.h`.
 
 ## How to choose where to add new specs - in TruffleRuby or in ruby/spec repository
 

doc/contributor/updating-ruby.md

@@ -142,7 +142,7 @@ Also update the list of `provided_executables` in `mx_truffleruby.py` if some la
 Update all of these:
 
 * Update `.ruby-version`, `TruffleRuby.LANGUAGE_VERSION`
-* Reset `lib/cext/ABI_version.txt` and `lib/cext/ABI_check.txt` to `1` if `RUBY_VERSION` was updated.
+* Reset `truffleruby-abi-version.h` to `$RUBY_VERSION.1` and `lib/cext/ABI_check.txt` to `1` if `RUBY_VERSION` was updated.
 * Update `versions.json` (with gem versions provided by `cat ../ruby/gems/bundled_gems | sort`, `ls -l lib/gems/specifications/default` and `grep 'VERSION =' lib/mri/rubygems.rb`)
 * Also update version numbers for `debug` and `rbs` in `src/main/c/Makefile` and in `mx.truffleruby/suite.py`.
 * Copy and paste `-h` and `--help` output to `RubyLauncher` (instructions are in the end of the file `src/launcher/java/org/truffleruby/launcher/RubyLauncher.java`)

lib/cext/ABI_version.txt

@@ -136,9 +136,13 @@ static inline bool BIGNUM_POSITIVE_P(VALUE b);
 static inline bool BIGNUM_NEGATIVE_P(VALUE b);
 static inline void BIGNUM_SET_SIGN(VALUE b, bool sign);
 static inline void BIGNUM_NEGATE(VALUE b);
+#ifndef TRUFFLERUBY
 static inline size_t BIGNUM_LEN(VALUE b);
+#endif
 static inline BDIGIT *BIGNUM_DIGITS(VALUE b);
+#ifndef TRUFFLERUBY
 static inline int BIGNUM_LENINT(VALUE b);
+#endif
 static inline bool BIGNUM_EMBED_P(VALUE b);
 
 RUBY_SYMBOL_EXPORT_BEGIN
@@ -205,6 +209,7 @@ BIGNUM_NEGATE(VALUE b)
     FL_REVERSE_RAW(b, BIGNUM_SIGN_BIT);
 }
 
+#ifndef TRUFFLERUBY
 static inline size_t
 BIGNUM_LEN(VALUE b)
 {
@@ -224,6 +229,7 @@ BIGNUM_LENINT(VALUE b)
 {
     return rb_long2int(BIGNUM_LEN(b));
 }
+#endif
 
 /* LSB:BIGNUM_DIGITS(b)[0], MSB:BIGNUM_DIGITS(b)[BIGNUM_LEN(b)-1] */
 static inline BDIGIT *

lib/cext/include/internal/bignum.h

@@ -25,7 +25,12 @@ st_index_t rb_iseq_cdhash_hash(VALUE a);
 /* iseq.c */
 int rb_vm_insn_addr2insn(const void *);
 int rb_vm_insn_decode(const VALUE encoded);
+
+#ifdef TRUFFLERUBY
+#define ruby_vm_keep_script_lines false
+#else
 extern bool ruby_vm_keep_script_lines;
+#endif
 
 MJIT_SYMBOL_EXPORT_BEGIN
 /* iseq.c (export) */

lib/cext/include/internal/compile.h

@@ -131,8 +131,10 @@ typedef struct rb_imemo_tmpbuf_struct rb_imemo_tmpbuf_t;
 rb_imemo_tmpbuf_t *rb_imemo_tmpbuf_parser_heap(void *buf, rb_imemo_tmpbuf_t *old_heap, size_t cnt);
 struct vm_ifunc *rb_vm_ifunc_new(rb_block_call_func_t func, const void *data, int min_argc, int max_argc);
 void rb_strterm_mark(VALUE obj);
+#ifndef TRUFFLERUBY
 static inline enum imemo_type imemo_type(VALUE imemo);
 static inline int imemo_type_p(VALUE imemo, enum imemo_type imemo_type);
+#endif
 static inline bool imemo_throw_data_p(VALUE imemo);
 static inline struct vm_ifunc *rb_vm_ifunc_proc_new(rb_block_call_func_t func, const void *data);
 #ifdef TRUFFLERUBY
@@ -160,6 +162,7 @@ VALUE rb_imemo_new(enum imemo_type type, VALUE v1, VALUE v2, VALUE v3, VALUE v0)
 const char *rb_imemo_name(enum imemo_type type);
 RUBY_SYMBOL_EXPORT_END
 
+#ifndef TRUFFLERUBY
 static inline enum imemo_type
 imemo_type(VALUE imemo)
 {
@@ -182,6 +185,7 @@ imemo_type_p(VALUE imemo, enum imemo_type imemo_type)
 }
 
 #define IMEMO_TYPE_P(v, t) imemo_type_p((VALUE)v, t)
+#endif
 
 static inline bool
 imemo_throw_data_p(VALUE imemo)

lib/cext/include/internal/imemo.h

@@ -106,10 +106,8 @@ int rb_long2int(long value);
 #endif
 RBIMPL_SYMBOL_EXPORT_END()
 
-#ifndef TRUFFLERUBY
 RBIMPL_ATTR_CONST_UNLESS_DEBUG()
 RBIMPL_ATTR_CONSTEXPR_UNLESS_DEBUG(CXX14)
-#endif
 RBIMPL_ATTR_ARTIFICIAL()
 /**
  * Converts a C's `long` into an instance of ::rb_cInteger.
@@ -231,10 +229,7 @@ rbimpl_right_shift_is_arithmetic_p(void)
 }
 
 RBIMPL_ATTR_CONST_UNLESS_DEBUG()
-#ifndef TRUFFLERUBY
 RBIMPL_ATTR_CONSTEXPR_UNLESS_DEBUG(CXX14)
-#endif
-
 /**
  * Converts a Fixnum into C's `long`.
  *
@@ -245,16 +240,12 @@ RBIMPL_ATTR_CONSTEXPR_UNLESS_DEBUG(CXX14)
 static inline long
 rb_fix2long(VALUE x)
 {
-#ifdef TRUFFLERUBY
-    return ((long)polyglot_as_i64(rb_tr_unwrap(x)));
-#else
     if /* constexpr */ (rbimpl_right_shift_is_arithmetic_p()) {
         return rbimpl_fix2long_by_shift(x);
     }
     else {
         return rbimpl_fix2long_by_idiv(x);
     }
-#endif
 }
 
 RBIMPL_ATTR_CONST_UNLESS_DEBUG()
@@ -344,11 +335,7 @@ rb_ulong2num_inline(unsigned long v)
     if (RB_POSFIXABLE(v))
         return RB_LONG2FIX(v);
     else
-#ifdef TRUFFLERUBY
-        return rb_tr_wrap(polyglot_invoke(RUBY_CEXT, "rb_ulong2num", (long) v));
-#else
         return rb_uint2big(v);
-#endif
 }
 
 /**

lib/cext/include/ruby/internal/arithmetic/long.h

@@ -56,6 +56,11 @@
 #  define HAVE_VA_ARGS_MACRO
 # else
 #  /* NG, not known. */
+// Always define HAVE_VA_ARGS_MACRO on TruffleRuby, as it leads to cleaner macros and more optimizations:
+// clang++ on macOS Ventura has __cplusplus defined as 199711 but it supports newer standards like c++11
+#ifdef TRUFFLERUBY
+#define HAVE_VA_ARGS_MACRO
+#endif
 # endif
 #endif
 
@@ -155,7 +160,7 @@
 #endif
 
 #define TRUFFLERUBY
-// Loaded at the end of config.h, included from defines.h. Needs STRINGIZE().
+// Loaded at the end of config.h, included from defines.h.
 #include <truffleruby/truffleruby-pre.h>
 
 #endif /* RBIMPL_CONFIG_H */

lib/cext/include/ruby/internal/config.h

@@ -283,6 +283,7 @@ VALUE rb_tr_rarray_aref(VALUE array, long index);
 #endif
 RBIMPL_SYMBOL_EXPORT_END()
 
+#ifndef TRUFFLERUBY
 RBIMPL_ATTR_PURE_UNLESS_DEBUG()
 RBIMPL_ATTR_ARTIFICIAL()
 /**
@@ -312,7 +313,6 @@ RARRAY_EMBED_LEN(VALUE ary)
     return RBIMPL_CAST((long)f);
 }
 
-#ifndef TRUFFLERUBY
 RBIMPL_ATTR_PURE_UNLESS_DEBUG()
 /**
  * Queries the length of the array.

lib/cext/include/ruby/internal/core/rarray.h

@@ -37,9 +37,7 @@
  * @param   obj  Arbitrary Ruby object.
  * @return  The passed object casted to ::RBasic.
  */
-#ifdef TRUFFLERUBY
-#define RBASIC(obj)                 (polyglot_as_RBasic(polyglot_invoke(RUBY_CEXT, "RBASIC", rb_tr_unwrap(obj))))
-#else
+#ifndef TRUFFLERUBY
 #define RBASIC(obj)                 RBIMPL_CAST((struct RBasic *)(obj))
 #endif
 
@@ -75,7 +73,8 @@ enum ruby_rvalue_flags {
 struct
 RUBY_ALIGNAS(SIZEOF_VALUE)
 RBasic {
-
+#ifndef TRUFFLERUBY
+    // TruffleRuby: we cannot support writing to the flags field, so don't expose the field
     /**
      * Per-object  flags.  Each  ruby  objects have  their own  characteristics
      * apart from their  classes.  For instance whether an object  is frozen or
@@ -98,11 +97,7 @@ RBasic {
      * Also note the `const` qualifier.  In  ruby an object cannot "change" its
      * class.
      */
-#ifdef TRUFFLERUBY
-    VALUE klass;
-#else
     const VALUE klass;
-#endif
 
 #ifdef __cplusplus
   public:
@@ -122,12 +117,9 @@ RBasic {
     {
     }
 #endif
+#endif // TRUFFLERUBY
 };
 
-#ifdef TRUFFLERUBY
-POLYGLOT_DECLARE_STRUCT(RBasic)
-#endif
-
 RBIMPL_SYMBOL_EXPORT_BEGIN()
 /**
  * Make the object invisible from Ruby code.