diff --git a/src/hotspot/cpu/aarch64/aarch64.ad b/src/hotspot/cpu/aarch64/aarch64.ad index 89e4abb8339..dd7d6e70720 100644 --- a/src/hotspot/cpu/aarch64/aarch64.ad +++ b/src/hotspot/cpu/aarch64/aarch64.ad @@ -2352,7 +2352,7 @@ bool Matcher::is_short_branch_offset(int rule, int br_size, int offset) { // Vector width in bytes. const int Matcher::vector_width_in_bytes(BasicType bt) { // The MaxVectorSize should have been set by detecting SVE max vector register size. - int size = MIN2((UseSVE > 0) ? 256 : 16, (int)MaxVectorSize); + int size = MIN2((UseSVE > 0) ? (int)FloatRegister::sve_vl_max : (int)FloatRegister::neon_vl, (int)MaxVectorSize); // Minimum 2 values in vector if (size < 2*type2aelembytes(bt)) size = 0; // But never < 4 @@ -2391,7 +2391,7 @@ const int Matcher::scalable_vector_reg_size(const BasicType bt) { // Vector ideal reg. const uint Matcher::vector_ideal_reg(int len) { - if (UseSVE > 0 && 16 < len && len <= 256) { + if (UseSVE > 0 && FloatRegister::neon_vl < len && len <= FloatRegister::sve_vl_max) { return Op_VecA; } switch(len) { diff --git a/src/hotspot/cpu/aarch64/aarch64_vector.ad b/src/hotspot/cpu/aarch64/aarch64_vector.ad index 18a36b011e4..bb4db56c8ba 100644 --- a/src/hotspot/cpu/aarch64/aarch64_vector.ad +++ b/src/hotspot/cpu/aarch64/aarch64_vector.ad @@ -154,7 +154,7 @@ source %{ } int length_in_bytes = vlen * type2aelembytes(bt); - if (UseSVE == 0 && length_in_bytes > 16) { + if (UseSVE == 0 && length_in_bytes > FloatRegister::neon_vl) { return false; } diff --git a/src/hotspot/cpu/aarch64/aarch64_vector_ad.m4 b/src/hotspot/cpu/aarch64/aarch64_vector_ad.m4 index 22dbcb9a549..ed190dbf524 100644 --- a/src/hotspot/cpu/aarch64/aarch64_vector_ad.m4 +++ b/src/hotspot/cpu/aarch64/aarch64_vector_ad.m4 @@ -144,7 +144,7 @@ source %{ } int length_in_bytes = vlen * type2aelembytes(bt); - if (UseSVE == 0 && length_in_bytes > 16) { + if (UseSVE == 0 && length_in_bytes > FloatRegister::neon_vl) { return false; } diff --git a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp index 02feaa02bf2..d40953c31b1 100644 --- a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp @@ -6002,8 +6002,10 @@ void MacroAssembler::cache_wbsync(bool is_pre) { } void MacroAssembler::verify_sve_vector_length(Register tmp) { + if (!UseSVE || VM_Version::get_max_supported_sve_vector_length() == FloatRegister::sve_vl_min) { + return; + } // Make sure that native code does not change SVE vector length. - if (!UseSVE) return; Label verify_ok; movw(tmp, zr); sve_inc(tmp, B); diff --git a/src/hotspot/cpu/aarch64/register_aarch64.hpp b/src/hotspot/cpu/aarch64/register_aarch64.hpp index 286dd83d687..eef4182efe9 100644 --- a/src/hotspot/cpu/aarch64/register_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/register_aarch64.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000, 2023, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2000, 2024, Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2014, 2021, Red Hat Inc. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * @@ -165,7 +165,13 @@ class FloatRegister { max_slots_per_register = 4, save_slots_per_register = 2, slots_per_neon_register = 4, - extra_save_slots_per_neon_register = slots_per_neon_register - save_slots_per_register + extra_save_slots_per_neon_register = slots_per_neon_register - save_slots_per_register, + neon_vl = 16, + // VLmax: The maximum sve vector length is determined by the hardware + // sve_vl_min <= VLmax <= sve_vl_max. + sve_vl_min = 16, + // Maximum supported vector length across all CPUs + sve_vl_max = 256 }; class FloatRegisterImpl: public AbstractRegisterImpl { diff --git a/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp b/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp index ee9fc80f0c5..c4aac20d005 100644 --- a/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/vm_version_aarch64.cpp @@ -25,6 +25,7 @@ #include "precompiled.hpp" #include "pauth_aarch64.hpp" +#include "register_aarch64.hpp" #include "runtime/arguments.hpp" #include "runtime/globals_extension.hpp" #include "runtime/java.hpp" @@ -44,6 +45,7 @@ int VM_Version::_zva_length; int VM_Version::_dcache_line_size; int VM_Version::_icache_line_size; int VM_Version::_initial_sve_vector_length; +int VM_Version::_max_supported_sve_vector_length; bool VM_Version::_rop_protection; uintptr_t VM_Version::_pac_mask; @@ -510,30 +512,37 @@ void VM_Version::initialize() { if (UseSVE > 0) { if (FLAG_IS_DEFAULT(MaxVectorSize)) { MaxVectorSize = _initial_sve_vector_length; - } else if (MaxVectorSize < 16) { - warning("SVE does not support vector length less than 16 bytes. Disabling SVE."); + } else if (MaxVectorSize < FloatRegister::sve_vl_min) { + warning("SVE does not support vector length less than %d bytes. Disabling SVE.", + FloatRegister::sve_vl_min); UseSVE = 0; - } else if ((MaxVectorSize % 16) == 0 && is_power_of_2(MaxVectorSize)) { - int new_vl = set_and_get_current_sve_vector_length(MaxVectorSize); - _initial_sve_vector_length = new_vl; - // Update MaxVectorSize to the largest supported value. - if (new_vl < 0) { - vm_exit_during_initialization( - err_msg("Current system does not support SVE vector length for MaxVectorSize: %d", - (int)MaxVectorSize)); - } else if (new_vl != MaxVectorSize) { - warning("Current system only supports max SVE vector length %d. Set MaxVectorSize to %d", - new_vl, new_vl); - } - MaxVectorSize = new_vl; - } else { + } else if (!((MaxVectorSize % FloatRegister::sve_vl_min) == 0 && is_power_of_2(MaxVectorSize))) { vm_exit_during_initialization(err_msg("Unsupported MaxVectorSize: %d", (int)MaxVectorSize)); } + + if (UseSVE > 0) { + // Acquire the largest supported vector length of this machine + _max_supported_sve_vector_length = set_and_get_current_sve_vector_length(FloatRegister::sve_vl_max); + + if (MaxVectorSize != _max_supported_sve_vector_length) { + int new_vl = set_and_get_current_sve_vector_length(MaxVectorSize); + if (new_vl < 0) { + vm_exit_during_initialization( + err_msg("Current system does not support SVE vector length for MaxVectorSize: %d", + (int)MaxVectorSize)); + } else if (new_vl != MaxVectorSize) { + warning("Current system only supports max SVE vector length %d. Set MaxVectorSize to %d", + new_vl, new_vl); + } + MaxVectorSize = new_vl; + } + _initial_sve_vector_length = MaxVectorSize; + } } if (UseSVE == 0) { // NEON int min_vector_size = 8; - int max_vector_size = 16; + int max_vector_size = FloatRegister::neon_vl; if (!FLAG_IS_DEFAULT(MaxVectorSize)) { if (!is_power_of_2(MaxVectorSize)) { vm_exit_during_initialization(err_msg("Unsupported MaxVectorSize: %d", (int)MaxVectorSize)); @@ -545,11 +554,11 @@ void VM_Version::initialize() { FLAG_SET_DEFAULT(MaxVectorSize, max_vector_size); } } else { - FLAG_SET_DEFAULT(MaxVectorSize, 16); + FLAG_SET_DEFAULT(MaxVectorSize, FloatRegister::neon_vl); } } - int inline_size = (UseSVE > 0 && MaxVectorSize >= 16) ? MaxVectorSize : 0; + int inline_size = (UseSVE > 0 && MaxVectorSize >= FloatRegister::sve_vl_min) ? MaxVectorSize : 0; if (FLAG_IS_DEFAULT(ArrayOperationPartialInlineSize)) { FLAG_SET_DEFAULT(ArrayOperationPartialInlineSize, inline_size); } else if (ArrayOperationPartialInlineSize != 0 && ArrayOperationPartialInlineSize != inline_size) { diff --git a/src/hotspot/cpu/aarch64/vm_version_aarch64.hpp b/src/hotspot/cpu/aarch64/vm_version_aarch64.hpp index c25c38a09c3..4b0975c558a 100644 --- a/src/hotspot/cpu/aarch64/vm_version_aarch64.hpp +++ b/src/hotspot/cpu/aarch64/vm_version_aarch64.hpp @@ -46,6 +46,7 @@ class VM_Version : public Abstract_VM_Version { static int _dcache_line_size; static int _icache_line_size; static int _initial_sve_vector_length; + static int _max_supported_sve_vector_length; static bool _rop_protection; static uintptr_t _pac_mask; @@ -164,7 +165,8 @@ enum Ampere_CPU_Model { static int icache_line_size() { return _icache_line_size; } static int dcache_line_size() { return _dcache_line_size; } - static int get_initial_sve_vector_length() { return _initial_sve_vector_length; }; + static int get_initial_sve_vector_length() { return _initial_sve_vector_length; }; + static int get_max_supported_sve_vector_length() { return _max_supported_sve_vector_length; }; static bool supports_fast_class_init_checks() { return true; } constexpr static bool supports_stack_watermark_barrier() { return true; }