From 6998824cda1220eac161be7ae2be21090dd87517 Mon Sep 17 00:00:00 2001 From: Elif Aslan Date: Tue, 6 Feb 2024 19:31:19 +0000 Subject: [PATCH] backport of :8324874: AArch64: crypto pmull based CRC32/CRC32C intrinsics clobber --- .../cpu/aarch64/macroAssembler_aarch64.cpp | 173 +++++++++--------- .../intrinsics/zip/TestFpRegsABI.java | 163 +++++++++++++++++ 2 files changed, 254 insertions(+), 82 deletions(-) create mode 100644 test/hotspot/jtreg/compiler/intrinsics/zip/TestFpRegsABI.java diff --git a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp index a3c560b28d3..5ab336d7d15 100644 --- a/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp +++ b/src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp @@ -4258,108 +4258,117 @@ void MacroAssembler::kernel_crc32_common_fold_using_crypto_pmull(Register crc, R } add(table, table, table_offset); + // Registers v0..v7 are used as data registers. + // Registers v16..v31 are used as tmp registers. sub(buf, buf, 0x10); - ldrq(v1, Address(buf, 0x10)); - ldrq(v2, Address(buf, 0x20)); - ldrq(v3, Address(buf, 0x30)); - ldrq(v4, Address(buf, 0x40)); - ldrq(v5, Address(buf, 0x50)); - ldrq(v6, Address(buf, 0x60)); - ldrq(v7, Address(buf, 0x70)); - ldrq(v8, Address(pre(buf, 0x80))); - - movi(v25, T4S, 0); - mov(v25, S, 0, crc); - eor(v1, T16B, v1, v25); - - ldrq(v0, Address(table)); + ldrq(v0, Address(buf, 0x10)); + ldrq(v1, Address(buf, 0x20)); + ldrq(v2, Address(buf, 0x30)); + ldrq(v3, Address(buf, 0x40)); + ldrq(v4, Address(buf, 0x50)); + ldrq(v5, Address(buf, 0x60)); + ldrq(v6, Address(buf, 0x70)); + ldrq(v7, Address(pre(buf, 0x80))); + + movi(v31, T4S, 0); + mov(v31, S, 0, crc); + eor(v0, T16B, v0, v31); + + // Register v16 contains constants from the crc table. + ldrq(v16, Address(table)); b(CRC_by128_loop); align(OptoLoopAlignment); BIND(CRC_by128_loop); - pmull (v9, T1Q, v1, v0, T1D); - pmull2(v10, T1Q, v1, v0, T2D); - ldrq(v1, Address(buf, 0x10)); - eor3(v1, T16B, v9, v10, v1); - - pmull (v11, T1Q, v2, v0, T1D); - pmull2(v12, T1Q, v2, v0, T2D); - ldrq(v2, Address(buf, 0x20)); - eor3(v2, T16B, v11, v12, v2); - - pmull (v13, T1Q, v3, v0, T1D); - pmull2(v14, T1Q, v3, v0, T2D); - ldrq(v3, Address(buf, 0x30)); - eor3(v3, T16B, v13, v14, v3); - - pmull (v15, T1Q, v4, v0, T1D); - pmull2(v16, T1Q, v4, v0, T2D); - ldrq(v4, Address(buf, 0x40)); - eor3(v4, T16B, v15, v16, v4); - - pmull (v17, T1Q, v5, v0, T1D); - pmull2(v18, T1Q, v5, v0, T2D); - ldrq(v5, Address(buf, 0x50)); - eor3(v5, T16B, v17, v18, v5); - - pmull (v19, T1Q, v6, v0, T1D); - pmull2(v20, T1Q, v6, v0, T2D); - ldrq(v6, Address(buf, 0x60)); - eor3(v6, T16B, v19, v20, v6); - - pmull (v21, T1Q, v7, v0, T1D); - pmull2(v22, T1Q, v7, v0, T2D); - ldrq(v7, Address(buf, 0x70)); - eor3(v7, T16B, v21, v22, v7); - - pmull (v23, T1Q, v8, v0, T1D); - pmull2(v24, T1Q, v8, v0, T2D); - ldrq(v8, Address(pre(buf, 0x80))); - eor3(v8, T16B, v23, v24, v8); + pmull (v17, T1Q, v0, v16, T1D); + pmull2(v18, T1Q, v0, v16, T2D); + ldrq(v0, Address(buf, 0x10)); + eor3(v0, T16B, v17, v18, v0); + + pmull (v19, T1Q, v1, v16, T1D); + pmull2(v20, T1Q, v1, v16, T2D); + ldrq(v1, Address(buf, 0x20)); + eor3(v1, T16B, v19, v20, v1); + + pmull (v21, T1Q, v2, v16, T1D); + pmull2(v22, T1Q, v2, v16, T2D); + ldrq(v2, Address(buf, 0x30)); + eor3(v2, T16B, v21, v22, v2); + + pmull (v23, T1Q, v3, v16, T1D); + pmull2(v24, T1Q, v3, v16, T2D); + ldrq(v3, Address(buf, 0x40)); + eor3(v3, T16B, v23, v24, v3); + + pmull (v25, T1Q, v4, v16, T1D); + pmull2(v26, T1Q, v4, v16, T2D); + ldrq(v4, Address(buf, 0x50)); + eor3(v4, T16B, v25, v26, v4); + + pmull (v27, T1Q, v5, v16, T1D); + pmull2(v28, T1Q, v5, v16, T2D); + ldrq(v5, Address(buf, 0x60)); + eor3(v5, T16B, v27, v28, v5); + + pmull (v29, T1Q, v6, v16, T1D); + pmull2(v30, T1Q, v6, v16, T2D); + ldrq(v6, Address(buf, 0x70)); + eor3(v6, T16B, v29, v30, v6); + + // Reuse registers v23, v24. + // Using them won't block the first instruction of the next iteration. + pmull (v23, T1Q, v7, v16, T1D); + pmull2(v24, T1Q, v7, v16, T2D); + ldrq(v7, Address(pre(buf, 0x80))); + eor3(v7, T16B, v23, v24, v7); subs(len, len, 0x80); br(Assembler::GE, CRC_by128_loop); // fold into 512 bits - ldrq(v0, Address(table, 0x10)); + // Use v31 for constants because v16 can be still in use. + ldrq(v31, Address(table, 0x10)); - pmull (v10, T1Q, v1, v0, T1D); - pmull2(v11, T1Q, v1, v0, T2D); - eor3(v1, T16B, v10, v11, v5); + pmull (v17, T1Q, v0, v31, T1D); + pmull2(v18, T1Q, v0, v31, T2D); + eor3(v0, T16B, v17, v18, v4); - pmull (v12, T1Q, v2, v0, T1D); - pmull2(v13, T1Q, v2, v0, T2D); - eor3(v2, T16B, v12, v13, v6); + pmull (v19, T1Q, v1, v31, T1D); + pmull2(v20, T1Q, v1, v31, T2D); + eor3(v1, T16B, v19, v20, v5); - pmull (v14, T1Q, v3, v0, T1D); - pmull2(v15, T1Q, v3, v0, T2D); - eor3(v3, T16B, v14, v15, v7); + pmull (v21, T1Q, v2, v31, T1D); + pmull2(v22, T1Q, v2, v31, T2D); + eor3(v2, T16B, v21, v22, v6); - pmull (v16, T1Q, v4, v0, T1D); - pmull2(v17, T1Q, v4, v0, T2D); - eor3(v4, T16B, v16, v17, v8); + pmull (v23, T1Q, v3, v31, T1D); + pmull2(v24, T1Q, v3, v31, T2D); + eor3(v3, T16B, v23, v24, v7); // fold into 128 bits - ldrq(v5, Address(table, 0x20)); - pmull (v10, T1Q, v1, v5, T1D); - pmull2(v11, T1Q, v1, v5, T2D); - eor3(v4, T16B, v4, v10, v11); - - ldrq(v6, Address(table, 0x30)); - pmull (v12, T1Q, v2, v6, T1D); - pmull2(v13, T1Q, v2, v6, T2D); - eor3(v4, T16B, v4, v12, v13); - - ldrq(v7, Address(table, 0x40)); - pmull (v14, T1Q, v3, v7, T1D); - pmull2(v15, T1Q, v3, v7, T2D); - eor3(v1, T16B, v4, v14, v15); + // Use v17 for constants because v31 can be still in use. + ldrq(v17, Address(table, 0x20)); + pmull (v25, T1Q, v0, v17, T1D); + pmull2(v26, T1Q, v0, v17, T2D); + eor3(v3, T16B, v3, v25, v26); + + // Use v18 for constants because v17 can be still in use. + ldrq(v18, Address(table, 0x30)); + pmull (v27, T1Q, v1, v18, T1D); + pmull2(v28, T1Q, v1, v18, T2D); + eor3(v3, T16B, v3, v27, v28); + + // Use v19 for constants because v18 can be still in use. + ldrq(v19, Address(table, 0x40)); + pmull (v29, T1Q, v2, v19, T1D); + pmull2(v30, T1Q, v2, v19, T2D); + eor3(v0, T16B, v3, v29, v30); add(len, len, 0x80); add(buf, buf, 0x10); - mov(tmp0, v1, D, 0); - mov(tmp1, v1, D, 1); + mov(tmp0, v0, D, 0); + mov(tmp1, v0, D, 1); } SkipIfEqual::SkipIfEqual( diff --git a/test/hotspot/jtreg/compiler/intrinsics/zip/TestFpRegsABI.java b/test/hotspot/jtreg/compiler/intrinsics/zip/TestFpRegsABI.java new file mode 100644 index 00000000000..46ecb64096b --- /dev/null +++ b/test/hotspot/jtreg/compiler/intrinsics/zip/TestFpRegsABI.java @@ -0,0 +1,163 @@ +/* + * Copyright Amazon.com Inc. or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + + +/** + * @test TestFpRegsABI + * @bug 8324874 + * @summary ABI for the Arm 64-bit Architecture requires to preserve registers v8-v15 by a callee across subroutine calls + * + * @run main/othervm -XX:-TieredCompilation -Xbatch -XX:CompileCommand=inline,*::calcValue compiler.intrinsics.zip.TestFpRegsABI + * @run main/othervm -XX:-TieredCompilation -Xbatch -XX:CompileCommand=dontinline,*::calcValue compiler.intrinsics.zip.TestFpRegsABI + * @run main/othervm -XX:+TieredCompilation -XX:TieredStopAtLevel=1 -Xbatch -XX:CompileCommand=inline,*::calcValue compiler.intrinsics.zip.TestFpRegsABI + * @run main/othervm -XX:+TieredCompilation -XX:TieredStopAtLevel=1 -Xbatch -XX:CompileCommand=dontinline,*::calcValue compiler.intrinsics.zip.TestFpRegsABI + * @run main/othervm -Xbatch -XX:CompileCommand=inline,*::calcValue compiler.intrinsics.zip.TestFpRegsABI + * @run main/othervm -Xbatch -XX:CompileCommand=dontinline,*::calcValue compiler.intrinsics.zip.TestFpRegsABI + * @run main/othervm -Xint compiler.intrinsics.zip.TestFpRegsABI + */ + +package compiler.intrinsics.zip; + +import java.util.zip.Checksum; +import java.util.zip.CRC32; +import java.util.zip.CRC32C; + +public class TestFpRegsABI { + private static byte[] buf; + + static { + buf = new byte[1024]; + for (int i = 0; i < buf.length; ++i) { + buf[i] = (byte)i; + } + } + + private static class RegressionTest { + Checksum checksum; + + RegressionTest(Checksum checksum) { + this.checksum = checksum; + } + + public void run(byte[] buf, long expectedValue) { + for (int i = 0; i < 20_000; ++i) { + runIteration(buf, expectedValue); + } + } + + // If checksum intrinsic does not save fp registers as ABI requires, + // the second call of calcValue might produce a wrong result. + private void runIteration(byte[] buf, long expectedValue) { + int v1 = calcValue(buf); + checksum.reset(); + checksum.update(buf, 0, buf.length); + long checksumValue = checksum.getValue(); + if (checksumValue != expectedValue) { + System.err.printf("ERROR: checksum = 0x%016x, expected = 0x%016x\n", + checksumValue, expectedValue); + throw new RuntimeException("Checksum Error"); + } + int v2 = calcValue(buf); + if (v1 != v2) { + throw new RuntimeException("Expect v2(" + v2 + ") to equal v1(" + v1 + ")"); + } + } + + private int calcValue(byte[] buf) { + return (int)(2.5 * buf.length); + } + } + + private static class TestIntrinsic { + Checksum checksum; + + TestIntrinsic(Checksum checksum) { + this.checksum = checksum; + } + + public void run(byte[] buf, long expectedValue) { + for (int i = 0; i < 20_000; ++i) { + runIteration(buf, expectedValue); + } + } + + // If checksum intrinsic does not save fp registers as ABI requires, + // the second call of calcValue might produce a wrong result. + private void runIteration(byte[] buf, long expectedValue) { + int v1 = calcValue(buf); + checksum.reset(); + checksum.update(buf, 0, buf.length); + long checksumValue = checksum.getValue(); + if (checksumValue != expectedValue) { + System.err.printf("ERROR: checksum = 0x%016x, expected = 0x%016x\n", + checksumValue, expectedValue); + throw new RuntimeException("Checksum Error"); + } + int v2 = calcValue(buf); + if (v1 != v2) { + throw new RuntimeException("Expect v2(" + v2 + ") to equal v1(" + v1 + ")"); + } + } + + // ABI can require some fp registers to be saved by a callee, e.g. v8-15 in ARM64 ABI. + // We create fp register pressure to get as many fp registers used as possible. + private int calcValue(byte[] buf) { + double v = 0.0; + for (int i = 24; i <= buf.length; i += 24) { + v += buf[i - 1] * ((double)i - 1.0) + (double)i - 1.0; + v += buf[i - 2] * ((double)i - 2.0) + (double)i - 2.0; + v += buf[i - 3] * ((double)i - 3.0) + (double)i - 3.0; + v += buf[i - 4] * ((double)i - 4.0) + (double)i - 4.0; + v += buf[i - 5] * ((double)i - 5.0) + (double)i - 5.0; + v += buf[i - 6] * ((double)i - 6.0) + (double)i - 6.0; + v += buf[i - 7] * ((double)i - 7.0) + (double)i - 7.0; + v += buf[i - 8] * ((double)i - 8.0) + (double)i - 8.0; + v += buf[i - 9] * ((double)i - 9.0) + (double)i - 9.0; + v += buf[i - 10] * ((double)i - 10.0) + (double)i - 10.0; + v += buf[i - 11] * ((double)i - 11.0) + (double)i - 11.0; + v += buf[i - 12] * ((double)i - 12.0) + (double)i - 12.0; + v += buf[i - 13] * ((double)i - 13.0) + (double)i - 13.0; + v += buf[i - 14] * ((double)i - 14.0) + (double)i - 14.0; + v += buf[i - 15] * ((double)i - 15.0) + (double)i - 15.0; + v += buf[i - 16] * ((double)i - 16.0) + (double)i - 16.0; + v += buf[i - 17] * ((double)i - 17.0) + (double)i - 17.0; + v += buf[i - 18] * ((double)i - 18.0) + (double)i - 18.0; + v += buf[i - 19] * ((double)i - 19.0) + (double)i - 19.0; + v += buf[i - 20] * ((double)i - 20.0) + (double)i - 20.0; + v += buf[i - 21] * ((double)i - 21.0) + (double)i - 21.0; + v += buf[i - 22] * ((double)i - 22.0) + (double)i - 22.0; + v += buf[i - 23] * ((double)i - 23.0) + (double)i - 23.0; + v += buf[i - 24] * ((double)i - 24.0) + (double)i - 24.0; + } + return (int)v; + } + } + + public static void main(final String[] argv) { + new TestIntrinsic(new CRC32()).run(buf, 0x00000000b70b4c26L); + new TestIntrinsic(new CRC32C()).run(buf, 0x000000002cdf6e8fL); + new RegressionTest(new CRC32()).run(buf, 0x00000000b70b4c26L); + new RegressionTest(new CRC32C()).run(buf, 0x000000002cdf6e8fL); + } +} +