Fix rb_integer_pack() for values between 2**62 and 2**63

* Fixes #3352
* Add specs for it.

Author: eregon

lib/truffle/truffle/cext.rb

@@ -844,6 +844,10 @@ def rb_intern(str)
     Primitive.string_to_symbol(str, true)
   end
 
+  def rb_int_cmp(a, b)
+    a <=> b
+  end
+
   def rb_int_positive_pow(a, b)
     a ** b
   end

spec/ruby/optional/capi/integer_spec.rb

@@ -140,6 +140,23 @@
               result.should == -1
               @words.should == "\x11\x32\x54\x76\x98\xBA\xDC\xFE"
             end
+
+            it "converts numbers near the fixnum limit successfully" do
+              result = @s.rb_integer_pack(0x7123_4567_89ab_cdef, @words, 1, 8, 0,
+                CApiIntegerSpecs::NATIVE|CApiIntegerSpecs::PACK_2COMP)
+              result.should == 1
+              @words.should == "\xEF\xCD\xAB\x89\x67\x45\x23\x71"
+
+              result = @s.rb_integer_pack(2**62-1, @words, 1, 8, 0,
+                CApiIntegerSpecs::NATIVE|CApiIntegerSpecs::PACK_2COMP)
+              result.should == 1
+              @words.should == "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x3F"
+
+              result = @s.rb_integer_pack(2**63-1, @words, 1, 8, 0,
+                CApiIntegerSpecs::NATIVE|CApiIntegerSpecs::PACK_2COMP)
+              result.should == 1
+              @words.should == "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x7F"
+            end
           end
         end
       end

src/main/c/cext/integer.c

@@ -101,7 +101,7 @@ int rb_integer_pack(VALUE value, void *words, size_t numwords, size_t wordsize,
     long l = NUM2LONG(value);
     sign = (l > 0) - (l < 0);
   } else {
-    sign = polyglot_as_i32(polyglot_invoke(rb_tr_unwrap(value), "<=>", 0));
+    sign = polyglot_as_i32(RUBY_CEXT_INVOKE_NO_WRAP("rb_int_cmp", value, INT2FIX(0)));
   }
   int bytes_needed = size / 8 + (size % 8 == 0 ? 0 : 1);
   int words_needed = bytes_needed / wordsize + (bytes_needed % wordsize == 0 ? 0 : 1);