8213538: VM crashes when MaxVectorSize is set to 0, 1 or 2

Require MaxVectorSize minimum 4 on 64 bit Reviewed-by: kvn, thartmann
openjdk · Jun 18, 2024 · 400cd83 · 400cd83
1 parent ef08f4e
commit 400cd83
Showing 1 changed file with 40 additions and 34 deletions.
diff --git a/src/hotspot/cpu/x86/vm_version_x86.cpp b/src/hotspot/cpu/x86/vm_version_x86.cpp
@@ -1056,44 +1056,50 @@ void VM_Version::get_processor_features() {
     }
   }
 #endif
+
 #if COMPILER2_OR_JVMCI
-  if (MaxVectorSize > 0) {
-    if (!is_power_of_2(MaxVectorSize)) {
-      warning("MaxVectorSize must be a power of 2");
-      FLAG_SET_DEFAULT(MaxVectorSize, 64);
-    }
-    if (UseSSE < 2) {
-      // Vectors (in XMM) are only supported with SSE2+
-      if (MaxVectorSize > 0) {
-        if (!FLAG_IS_DEFAULT(MaxVectorSize))
-          warning("MaxVectorSize must be 0");
-        FLAG_SET_DEFAULT(MaxVectorSize, 0);
-      }
-    }
-    else if (UseAVX == 0 || !os_supports_avx_vectors()) {
-      // 32 bytes vectors (in YMM) are only supported with AVX+
-      if (MaxVectorSize > 16) {
-        if (!FLAG_IS_DEFAULT(MaxVectorSize))
-          warning("MaxVectorSize must be <= 16");
-        FLAG_SET_DEFAULT(MaxVectorSize, 16);
-      }
+  int max_vector_size = 0;
+  if (UseSSE < 2) {
+    // Vectors (in XMM) are only supported with SSE2+
+    // SSE is always 2 on x64.
+    max_vector_size = 0;
+  } else if (UseAVX == 0 || !os_supports_avx_vectors()) {
+    // 16 byte vectors (in XMM) are supported with SSE2+
+    max_vector_size = 16;
+  } else if (UseAVX == 1 || UseAVX == 2) {
+    // 32 bytes vectors (in YMM) are only supported with AVX+
+    max_vector_size = 32;
+  } else if (UseAVX > 2 ) {
+    // 64 bytes vectors (in ZMM) are only supported with AVX 3
+    max_vector_size = 64;
+  }
+
+#ifdef _LP64
+  int min_vector_size = 4; // We require MaxVectorSize to be at least 4 on 64bit
+#else
+  int min_vector_size = 0;
+#endif
+
+  if (!FLAG_IS_DEFAULT(MaxVectorSize)) {
+    if (MaxVectorSize < min_vector_size) {
+      warning("MaxVectorSize must be at least %i on this platform", min_vector_size);
+      FLAG_SET_DEFAULT(MaxVectorSize, min_vector_size);
     }
-    else if (UseAVX == 1 || UseAVX == 2) {
-      // 64 bytes vectors (in ZMM) are only supported with AVX 3
-      if (MaxVectorSize > 32) {
-        if (!FLAG_IS_DEFAULT(MaxVectorSize))
-          warning("MaxVectorSize must be <= 32");
-        FLAG_SET_DEFAULT(MaxVectorSize, 32);
-      }
+    if (MaxVectorSize > max_vector_size) {
+      warning("MaxVectorSize must be at most %i on this platform", max_vector_size);
+      FLAG_SET_DEFAULT(MaxVectorSize, max_vector_size);
     }
-    else if (UseAVX > 2 ) {
-      if (MaxVectorSize > 64) {
-        if (!FLAG_IS_DEFAULT(MaxVectorSize))
-          warning("MaxVectorSize must be <= 64");
-        FLAG_SET_DEFAULT(MaxVectorSize, 64);
-      }
+    if (!is_power_of_2(MaxVectorSize)) {
+      warning("MaxVectorSize must be a power of 2, setting to default: %i", max_vector_size);
+      FLAG_SET_DEFAULT(MaxVectorSize, max_vector_size);
     }
+  } else {
+    // If default, use highest supported configuration
+    FLAG_SET_DEFAULT(MaxVectorSize, max_vector_size);
+  }
+
 #if defined(COMPILER2) && defined(ASSERT)
+  if (MaxVectorSize > 0) {
     if (supports_avx() && PrintMiscellaneous && Verbose && TraceNewVectors) {
       tty->print_cr("State of YMM registers after signal handle:");
       int nreg = 2 LP64_ONLY(+2);
@@ -1106,8 +1112,8 @@ void VM_Version::get_processor_features() {
         tty->cr();
       }
     }
-#endif // COMPILER2 && ASSERT
   }
+#endif // COMPILER2 && ASSERT
 #endif // COMPILER2_OR_JVMCI
 
 #ifdef COMPILER2