Ensure JIT_Dbl2ULng correctly handles NaN

tannergooding · May 27, 2022 · 3298345 · 3298345
1 parent 5d5fb3a
commit 3298345
Showing 1 changed file with 5 additions and 1 deletion.
diff --git a/src/coreclr/vm/jithelpers.cpp b/src/coreclr/vm/jithelpers.cpp
@@ -595,10 +595,14 @@ HCIMPL1_V(UINT64, JIT_Dbl2ULng, double val)
     if (val < two63) {
         ret = FastDbl2Lng(val);
     }
-    else {
+    else if (!_isnan(val)) {
         // subtract 0x8000000000000000, do the convert then add it back again
         ret = FastDbl2Lng(val - two63) + I64(0x8000000000000000);
     }
+    else {
+        // specially handle NaN otherwise we return (Int64.MaxValue + 1)
+        ret = 0;
+    }
     return ret;
 }
 HCIMPLEND