From 8404d4fdfc5be0ecac158dc64f83a3b9c9a45f19 Mon Sep 17 00:00:00 2001
From: Benoit Daloze <benoit.daloze@oracle.com>
Date: Fri, 20 Oct 2023 15:37:14 +0200
Subject: [PATCH] Fix rb_type() for values >= 1**62 and < 1**63 and
 corresponding negative values

* So rb_type(value) == T_FIXNUM is equivalent to RB_FIXNUM_P().
* Fix related rb_big* functions so they handle such values (which are not RubyBignum objects).
---
 lib/truffle/truffle/cext.rb | 17 ++++++++++++++++-
 src/main/c/cext/integer.c   |  4 ++--
 src/main/c/cext/numeric.c   |  8 ++++----
 3 files changed, 22 insertions(+), 7 deletions(-)

diff --git a/lib/truffle/truffle/cext.rb b/lib/truffle/truffle/cext.rb
index 70afb5d1a79..38a937708d5 100644
--- a/lib/truffle/truffle/cext.rb
+++ b/lib/truffle/truffle/cext.rb
@@ -102,6 +102,9 @@ module Truffle::CExt
 
   T_MASK     = 0x1f
 
+  RUBY_FIXNUM_MIN = -(1 << 62)
+  RUBY_FIXNUM_MAX = (1 << 62) - 1
+
   # This list of types is derived from MRI's error.c builtin_types array.
   BUILTIN_TYPES = [
     '',
@@ -276,7 +279,11 @@ def rb_tr_cached_type(value, type)
         T_OBJECT
       end
     elsif type == T_FIXNUM
-      Truffle::Type.fits_into_long?(value) ? T_FIXNUM : T_BIGNUM
+      if RUBY_FIXNUM_MIN <= value && value <= RUBY_FIXNUM_MAX
+        T_FIXNUM
+      else
+        T_BIGNUM
+      end
     else
       type
     end
@@ -427,6 +434,14 @@ def rb_num2long(val)
     Primitive.rb_num2long(val)
   end
 
+  def rb_big_sign(val)
+    val >= 0
+  end
+
+  def rb_big_cmp(x, y)
+    x <=> y
+  end
+
   def rb_big2dbl(val)
     Truffle::Type.rb_big2dbl(val)
   end
diff --git a/src/main/c/cext/integer.c b/src/main/c/cext/integer.c
index dbaf5e34724..91bb3ed6cea 100644
--- a/src/main/c/cext/integer.c
+++ b/src/main/c/cext/integer.c
@@ -136,7 +136,7 @@ size_t rb_absint_size(VALUE value, int *nlz_bits_ret) {
 }
 
 int rb_big_sign(VALUE x) {
-  return RTEST(RUBY_INVOKE(x, ">=", INT2FIX(0))) ? 1 : 0;
+  return RTEST(RUBY_CEXT_INVOKE("rb_big_sign", x)) ? 1 : 0;
 }
 
 int rb_cmpint(VALUE val, VALUE a, VALUE b) {
@@ -144,7 +144,7 @@ int rb_cmpint(VALUE val, VALUE a, VALUE b) {
 }
 
 VALUE rb_big_cmp(VALUE x, VALUE y) {
-  return RUBY_INVOKE(x, "<=>", y);
+  return RUBY_CEXT_INVOKE("rb_big_cmp", x, y);
 }
 
 void rb_big_pack(VALUE val, unsigned long *buf, long num_longs) {
diff --git a/src/main/c/cext/numeric.c b/src/main/c/cext/numeric.c
index cb02ae431a3..aead1875d9b 100644
--- a/src/main/c/cext/numeric.c
+++ b/src/main/c/cext/numeric.c
@@ -230,10 +230,6 @@ long rb_big2long(VALUE x) {
   return polyglot_as_i64(RUBY_CEXT_INVOKE_NO_WRAP("rb_num2long", x));
 }
 
-VALUE rb_big2str(VALUE x, int base) {
-  return rb_tr_wrap(polyglot_invoke(rb_tr_unwrap(x), "to_s", base));
-}
-
 unsigned long rb_big2ulong(VALUE x) {
   return polyglot_as_i64(RUBY_CEXT_INVOKE_NO_WRAP("rb_num2ulong", x));
 }
@@ -262,6 +258,10 @@ VALUE rb_fix2str(VALUE x, int base) {
   return RUBY_CEXT_INVOKE("rb_fix2str", x, INT2FIX(base));
 }
 
+VALUE rb_big2str(VALUE x, int base) {
+  return rb_fix2str(x, base);
+}
+
 VALUE rb_to_int(VALUE object) {
   return RUBY_CEXT_INVOKE("rb_to_int", object);
 }