8256267: Relax compiler/floatingpoint/NaNTest.java for x86_32 and lower -XX:+UseSSE

Reviewed-by: kvn, iignatyev

Author: shipilev

test/hotspot/jtreg/compiler/floatingpoint/NaNTest.java

@@ -24,13 +24,23 @@
  * @test
  * @bug 8076373
  * @summary Verify if signaling NaNs are preserved.
+ * @library /test/lib /
  *
- * @run main compiler.floatingpoint.NaNTest
+ * @build sun.hotspot.WhiteBox
+ * @run driver ClassFileInstaller sun.hotspot.WhiteBox
+ * @run main/othervm -Xbootclasspath/a:. -XX:+UnlockDiagnosticVMOptions -XX:+WhiteBoxAPI
+ *                   compiler.floatingpoint.NaNTest
  */
 
 package compiler.floatingpoint;
 
+import jdk.test.lib.Platform;
+import jtreg.SkippedException;
+import sun.hotspot.WhiteBox;
+
 public class NaNTest {
+    static final WhiteBox WHITE_BOX = WhiteBox.getWhiteBox();
+
     static void testFloat() {
         int originalValue = 0x7f800001;
         int readBackValue = Float.floatToRawIntBits(Float.intBitsToFloat(originalValue));
@@ -63,11 +73,34 @@ static void testDouble() {
     }
 
     public static void main(String args[]) {
-        System.out.println("### NanTest started");
+        // Some platforms are known to strip signaling NaNs.
+        // The block below can be used to except them.
+        boolean expectStableFloats = true;
+        boolean expectStableDoubles = true;
+
+        // On x86_32 without relevant SSE-enabled stubs, we are entering
+        // native methods that use FPU instructions, and those strip the
+        // signaling NaNs.
+        if (Platform.isX86()) {
+            int sse = WHITE_BOX.getIntxVMFlag("UseSSE").intValue();
+            expectStableFloats = (sse >= 1);
+            expectStableDoubles = (sse >= 2);
+        }
+
+        if (expectStableFloats) {
+           testFloat();
+        } else {
+           System.out.println("Stable floats cannot be expected, skipping");
+        }
 
-        testFloat();
-        testDouble();
+        if (expectStableDoubles) {
+           testDouble();
+        } else {
+           System.out.println("Stable doubles cannot be expected, skipping");
+        }
 
-        System.out.println("### NanTest ended");
+        if (!expectStableFloats && !expectStableDoubles) {
+           throw new SkippedException("No tests were run.");
+        }
     }
 }