From 03ea75894f55230a5ced0cfbd1b57c795e83a50a Mon Sep 17 00:00:00 2001 From: Tiago Alves Date: Fri, 27 Sep 2024 22:45:52 +0100 Subject: [PATCH 01/17] Enables access to FileTime private field When running tests with JDK17 we see NoClassDefFoundError when trying to instante org.apache.cassandra.Util class due caused by getSupportedMTimeGranularity trying to access a private field. This patch modified the build configuration to add a jvm option allowing that access. --- build.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/build.xml b/build.xml index 3f9fecb6c77a..bd646f9e0c4f 100644 --- a/build.xml +++ b/build.xml @@ -308,6 +308,7 @@ --add-opens java.base/java.lang=ALL-UNNAMED --add-opens java.base/java.util=ALL-UNNAMED --add-opens java.base/java.nio=ALL-UNNAMED + --add-opens java.base/java.nio.file.attribute=ALL-UNNAMED --add-opens java.rmi/sun.rmi.transport.tcp=ALL-UNNAMED From ab675c43cb357d8d0f6d416f09bbb70e9dfc6289 Mon Sep 17 00:00:00 2001 From: Ekaterina Dimitrova Date: Mon, 10 Jun 2024 10:06:51 +0300 Subject: [PATCH 02/17] Fix tests post JDK-8210522 (rewrite reflection of modifiers field) --- test/unit/org/apache/cassandra/index/sai/SAITester.java | 3 ++- test/unit/org/apache/cassandra/index/sai/SAIUtil.java | 3 ++- .../apache/cassandra/index/sai/cql/VectorUpdateDeleteTest.java | 1 + .../cassandra/index/sai/disk/io/TrackingIndexFileUtils.java | 1 + .../cassandra/index/sai/memory/TrieMemtableIndexTest.java | 3 ++- 5 files changed, 8 insertions(+), 3 deletions(-) diff --git a/test/unit/org/apache/cassandra/index/sai/SAITester.java b/test/unit/org/apache/cassandra/index/sai/SAITester.java index d1d46d1f7c18..ce82c9fafd66 100644 --- a/test/unit/org/apache/cassandra/index/sai/SAITester.java +++ b/test/unit/org/apache/cassandra/index/sai/SAITester.java @@ -92,6 +92,7 @@ import org.apache.cassandra.service.StorageService; import org.apache.cassandra.service.snapshot.TableSnapshot; import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.utils.ReflectionUtils; import org.apache.cassandra.utils.Throwables; import org.apache.lucene.codecs.CodecUtil; import org.awaitility.Awaitility; @@ -898,7 +899,7 @@ protected static void setSegmentWriteBufferSpace(final int segmentSize) throws E NamedMemoryLimiter limiter = (NamedMemoryLimiter) V1OnDiskFormat.class.getDeclaredField("SEGMENT_BUILD_MEMORY_LIMITER").get(null); Field limitBytes = limiter.getClass().getDeclaredField("limitBytes"); limitBytes.setAccessible(true); - Field modifiersField = Field.class.getDeclaredField("modifiers"); + Field modifiersField = ReflectionUtils.getField(Field.class, "modifiers"); modifiersField.setAccessible(true); modifiersField.setInt(limitBytes, limitBytes.getModifiers() & ~Modifier.FINAL); limitBytes.set(limiter, segmentSize); diff --git a/test/unit/org/apache/cassandra/index/sai/SAIUtil.java b/test/unit/org/apache/cassandra/index/sai/SAIUtil.java index 8f56a0f4ca9b..f7580332077c 100644 --- a/test/unit/org/apache/cassandra/index/sai/SAIUtil.java +++ b/test/unit/org/apache/cassandra/index/sai/SAIUtil.java @@ -21,6 +21,7 @@ import java.lang.reflect.Field; import org.apache.cassandra.index.sai.disk.format.Version; +import org.apache.cassandra.utils.ReflectionUtils; public class SAIUtil { @@ -31,7 +32,7 @@ public static void setLatestVersion(Version version) { latest = Version.class.getDeclaredField("LATEST"); latest.setAccessible(true); - Field modifiersField = Field.class.getDeclaredField("modifiers"); + Field modifiersField = ReflectionUtils.getField(Field.class, "modifiers"); modifiersField.setAccessible(true); latest.set(null, version); } diff --git a/test/unit/org/apache/cassandra/index/sai/cql/VectorUpdateDeleteTest.java b/test/unit/org/apache/cassandra/index/sai/cql/VectorUpdateDeleteTest.java index 5b74719c87fb..97f4bc5ef2f3 100644 --- a/test/unit/org/apache/cassandra/index/sai/cql/VectorUpdateDeleteTest.java +++ b/test/unit/org/apache/cassandra/index/sai/cql/VectorUpdateDeleteTest.java @@ -34,6 +34,7 @@ import org.apache.cassandra.index.sai.disk.format.Version; import org.apache.cassandra.index.sai.disk.vector.VectorMemtableIndex; import org.apache.cassandra.index.sai.plan.QueryController; +import org.apache.cassandra.utils.ReflectionUtils; import static org.apache.cassandra.index.sai.cql.VectorTypeTest.assertContainsInt; import static org.apache.cassandra.index.sai.disk.vector.CassandraOnHeapGraph.MIN_PQ_ROWS; diff --git a/test/unit/org/apache/cassandra/index/sai/disk/io/TrackingIndexFileUtils.java b/test/unit/org/apache/cassandra/index/sai/disk/io/TrackingIndexFileUtils.java index be9bca5c3300..53ed1295e899 100644 --- a/test/unit/org/apache/cassandra/index/sai/disk/io/TrackingIndexFileUtils.java +++ b/test/unit/org/apache/cassandra/index/sai/disk/io/TrackingIndexFileUtils.java @@ -29,6 +29,7 @@ import org.apache.cassandra.io.util.FileHandle; import org.apache.cassandra.io.util.SequentialWriterOption; +import org.apache.cassandra.utils.ReflectionUtils; public class TrackingIndexFileUtils extends IndexFileUtils { diff --git a/test/unit/org/apache/cassandra/index/sai/memory/TrieMemtableIndexTest.java b/test/unit/org/apache/cassandra/index/sai/memory/TrieMemtableIndexTest.java index b6995a64ba88..8184bea22c8c 100644 --- a/test/unit/org/apache/cassandra/index/sai/memory/TrieMemtableIndexTest.java +++ b/test/unit/org/apache/cassandra/index/sai/memory/TrieMemtableIndexTest.java @@ -67,6 +67,7 @@ import org.apache.cassandra.service.StorageService; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.Pair; +import org.apache.cassandra.utils.ReflectionUtils; import org.apache.cassandra.utils.bytecomparable.ByteComparable; import org.apache.cassandra.utils.bytecomparable.ByteSource; import org.apache.cassandra.utils.concurrent.OpOrder; @@ -364,7 +365,7 @@ private void setTrieMemtableBufferType(final BufferType newBufferType) throws Ex { Field bufferType = TrieMemtable.class.getDeclaredField("BUFFER_TYPE"); bufferType.setAccessible(true); - Field modifiersField = Field.class.getDeclaredField("modifiers"); + Field modifiersField = ReflectionUtils.getField(Field.class, "modifiers"); modifiersField.setAccessible(true); modifiersField.setInt(bufferType, bufferType.getModifiers() & ~Modifier.FINAL); bufferType.set(null, newBufferType); From fcc2381cbd54bac17e874e1e07ea7bad12f700c0 Mon Sep 17 00:00:00 2001 From: Ekaterina Dimitrova Date: Mon, 30 Sep 2024 14:16:32 -0400 Subject: [PATCH 03/17] JDK 22 config added --- build.xml | 102 ++++++++++++++++++++++++++++- conf/jvm22-clients.options | 31 +++++++++ conf/jvm22-server.options | 127 +++++++++++++++++++++++++++++++++++++ ide/idea/workspace.xml | 3 + 4 files changed, 262 insertions(+), 1 deletion(-) create mode 100644 conf/jvm22-clients.options create mode 100644 conf/jvm22-server.options diff --git a/build.xml b/build.xml index bd646f9e0c4f..e657ec72ff01 100644 --- a/build.xml +++ b/build.xml @@ -45,7 +45,7 @@ The use of both CASSANDRA_USE_JDK11 and use-jdk11 is deprecated. --> - + @@ -317,6 +317,78 @@ + + -XX:+UnlockDiagnosticVMOptions + -Djdk.attach.allowAttachSelf=true + -XX:+UseG1GC + -XX:+ParallelRefProcEnabled + + + -XX:G1RSetUpdatingPauseTimePercent=5 + -XX:MaxGCPauseMillis=100 + + + -XX:-RestrictContended + -XX:+UseThreadPriorities + -XX:+DebugNonSafepoints + -XX:+UseStringDeduplication + -XX:StringTableSize=1000003 + -XX:+PerfDisableSharedMem + -XX:+AlwaysPreTouch + -XX:+UseTLAB + -XX:+ResizeTLAB + -XX:+UseNUMA + + + --add-exports java.base/jdk.internal.misc=ALL-UNNAMED + --add-exports java.base/jdk.internal.ref=ALL-UNNAMED + --add-exports java.base/jdk.internal.perf=ALL-UNNAMED + --add-exports java.base/sun.nio.ch=ALL-UNNAMED + --add-exports java.management.rmi/com.sun.jmx.remote.internal.rmi=ALL-UNNAMED + --add-exports java.rmi/sun.rmi.registry=ALL-UNNAMED + --add-exports java.rmi/sun.rmi.server=ALL-UNNAMED + --add-exports jdk.naming.dns/com.sun.jndi.dns=java.naming + --add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED + --add-exports jdk.unsupported/sun.misc=ALL-UNNAMED + --add-exports java.base/jdk.internal.util.jar=ALL-UNNAMED + + --add-opens java.base/jdk.internal.module=ALL-UNNAMED + --add-opens java.base/jdk.internal.reflect=ALL-UNNAMED + --add-opens java.base/jdk.internal.loader=ALL-UNNAMED + --add-opens java.base/jdk.internal.ref=ALL-UNNAMED + --add-opens java.base/jdk.internal.math=ALL-UNNAMED + --add-opens java.base/java.math=ALL-UNNAMED + --add-opens jdk.management.jfr/jdk.management.jfr=ALL-UNNAMED + --add-opens jdk.management/com.sun.management.internal=ALL-UNNAMED + --add-opens jdk.naming.dns/com.sun.jndi.dns=ALL-UNNAMED + --add-opens java.base/java.lang.module=ALL-UNNAMED + --add-opens java.base/java.net=ALL-UNNAMED + --add-opens java.base/java.io=ALL-UNNAMED + --add-opens java.base/sun.nio.ch=ALL-UNNAMED + --add-opens java.base/java.nio=ALL-UNNAMED + --add-opens java.base/java.nio.file.spi=ALL-UNNAMED + --add-opens java.base/java.lang=ALL-UNNAMED + --add-opens java.base/java.util=ALL-UNNAMED + --add-opens java.base/java.lang.reflect=ALL-UNNAMED + --add-opens jdk.compiler/com.sun.tools.javac=ALL-UNNAMED + + + --add-opens java.base/java.nio.file.attribute=ALL-UNNAMED + + + + + --add-opens java.base/java.util.concurrent=ALL-UNNAMED + --add-opens java.base/java.util.concurrent.atomic=ALL-UNNAMED + + + + + + + --add-modules jdk.incubator.vector + + + + @@ -1251,6 +1332,7 @@ + @@ -1268,6 +1350,7 @@ + @@ -1293,6 +1376,7 @@ + @@ -1314,6 +1398,7 @@ + @@ -1337,6 +1422,7 @@ + @@ -1359,6 +1445,7 @@ + @@ -1376,6 +1463,7 @@ + @@ -1398,6 +1486,7 @@ + @@ -1414,6 +1503,7 @@ + @@ -1429,6 +1519,7 @@ + @@ -1459,6 +1550,7 @@ + @@ -1622,6 +1714,7 @@ + @@ -1668,6 +1761,7 @@ + @@ -1891,6 +1985,7 @@ + @@ -1902,6 +1997,7 @@ + @@ -1912,6 +2008,7 @@ + @@ -1938,6 +2035,7 @@ + @@ -1954,6 +2052,7 @@ + @@ -1967,6 +2066,7 @@ + diff --git a/conf/jvm22-clients.options b/conf/jvm22-clients.options new file mode 100644 index 000000000000..6e30351a77ee --- /dev/null +++ b/conf/jvm22-clients.options @@ -0,0 +1,31 @@ +########################################################################### +# jvm22-clients.options # +# # +# See jvm-clients.options. This file is specific for Java 22 and newer. # +########################################################################### + +################### +# JPMS SETTINGS # +################### + +-Djdk.attach.allowAttachSelf=true +--add-exports java.base/jdk.internal.misc=ALL-UNNAMED +--add-exports java.base/jdk.internal.ref=ALL-UNNAMED +--add-exports java.base/sun.nio.ch=ALL-UNNAMED +--add-exports java.management.rmi/com.sun.jmx.remote.internal.rmi=ALL-UNNAMED +--add-exports java.rmi/sun.rmi.registry=ALL-UNNAMED +--add-exports java.rmi/sun.rmi.server=ALL-UNNAMED +--add-exports java.sql/java.sql=ALL-UNNAMED + +--add-opens java.base/java.lang.module=ALL-UNNAMED +--add-opens java.base/jdk.internal.loader=ALL-UNNAMED +--add-opens java.base/jdk.internal.ref=ALL-UNNAMED +--add-opens java.base/jdk.internal.reflect=ALL-UNNAMED +--add-opens java.base/jdk.internal.math=ALL-UNNAMED +--add-opens java.base/jdk.internal.module=ALL-UNNAMED +--add-opens jdk.management/com.sun.management.internal=ALL-UNNAMED +--add-opens=java.base/java.lang.reflect=ALL-UNNAMED +--add-opens java.base/java.io=ALL-UNNAMED +--add-opens java.base/sun.nio.ch=ALL-UNNAMED + +# The newline in the end of file is intentional \ No newline at end of file diff --git a/conf/jvm22-server.options b/conf/jvm22-server.options new file mode 100644 index 000000000000..4917f3efcb8a --- /dev/null +++ b/conf/jvm22-server.options @@ -0,0 +1,127 @@ +########################################################################### +# jvm22-server.options # +# # +# See jvm-server.options. This file is specific for Java 22 and newer. # +########################################################################### + +################# +# GC SETTINGS # +################# + + + +### CMS Settings +#-XX:+UseConcMarkSweepGC +#-XX:+CMSParallelRemarkEnabled +#-XX:SurvivorRatio=8 +#-XX:MaxTenuringThreshold=1 +#-XX:CMSInitiatingOccupancyFraction=75 +#-XX:+UseCMSInitiatingOccupancyOnly +#-XX:CMSWaitDuration=10000 +#-XX:+CMSParallelInitialMarkEnabled +#-XX:+CMSEdenChunksRecordAlways +## some JVMs will fill up their heap when accessed via JMX, see CASSANDRA-6541 +#-XX:+CMSClassUnloadingEnabled + + + +### G1 Settings +## Use the Hotspot garbage-first collector. +-XX:+UseG1GC +-XX:+ParallelRefProcEnabled + +# +## Have the JVM do less remembered set work during STW, instead +## preferring concurrent GC. Reduces p99.9 latency. +-XX:G1RSetUpdatingPauseTimePercent=5 +# +## Main G1GC tunable: lowering the pause target will lower throughput and vise versa. +## 200ms is the JVM default and lowest viable setting +## 1000ms increases throughput. Keep it smaller than the timeouts in cassandra.yaml. +-XX:MaxGCPauseMillis=500 + +## Optional G1 Settings +# Save CPU time on large (>= 16GB) heaps by delaying region scanning +# until the heap is 70% full. The default in Hotspot 8u40 is 40%. +#-XX:InitiatingHeapOccupancyPercent=70 + +# For systems with > 8 cores, the default ParallelGCThreads is 5/8 the number of logical cores. +# Otherwise equal to the number of cores when 8 or less. +# Machines with > 10 cores should try setting these to <= full cores. +#-XX:ParallelGCThreads=16 +# By default, ConcGCThreads is 1/4 of ParallelGCThreads. +# Setting both to the same value can reduce STW durations. +#-XX:ConcGCThreads=16 + + +### JPMS + +-Djdk.attach.allowAttachSelf=true +--add-exports java.base/jdk.internal.misc=ALL-UNNAMED +--add-exports java.base/jdk.internal.ref=ALL-UNNAMED +--add-exports java.base/jdk.internal.perf=ALL-UNNAMED +--add-exports java.base/sun.nio.ch=ALL-UNNAMED +--add-exports java.management.rmi/com.sun.jmx.remote.internal.rmi=ALL-UNNAMED +--add-exports java.rmi/sun.rmi.registry=ALL-UNNAMED +--add-exports java.rmi/sun.rmi.server=ALL-UNNAMED +# --add-exports java.sql/java.sql=ALL-UNNAMED --> not seen in +# Below exist in the other script +--add-exports jdk.naming.dns/com.sun.jndi.dns=java.naming +--add-exports jdk.unsupported/sun.misc=ALL-UNNAMED +--add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED +--add-exports java.base/jdk.internal.util.jar=ALL-UNNAMED + +--add-opens java.base/jdk.internal.loader=ALL-UNNAMED +--add-opens java.base/jdk.internal.ref=ALL-UNNAMED +--add-opens java.base/jdk.internal.reflect=ALL-UNNAMED +--add-opens java.base/jdk.internal.math=ALL-UNNAMED +--add-opens java.base/jdk.internal.module=ALL-UNNAMED +--add-opens jdk.management/com.sun.management.internal=ALL-UNNAMED +# Added in +--add-opens java.base/java.lang.module=ALL-UNNAMED +--add-opens jdk.management.jfr/jdk.management.jfr=ALL-UNNAMED +--add-opens jdk.naming.dns/com.sun.jndi.dns=ALL-UNNAMED +--add-opens jdk.compiler/com.sun.tools.javac=ALL-UNNAMED +--add-opens java.base/java.nio.file.spi=ALL-UNNAMED +--add-opens java.base/java.net=ALL-UNNAMED +--add-opens java.base/jdk.internal.math=ALL-UNNAMED +--add-opens java.base/java.lang.reflect=ALL-UNNAMED + +# required for org.apache.cassandra.Util.getSupportedMTimeGranularity +--add-opens java.base/java.nio.file.attribute=ALL-UNNAMED + + +### GC logging options -- uncomment to enable + +# Java 11 (and newer) GC logging options: +# See description of https://bugs.openjdk.java.net/browse/JDK-8046148 for details about the syntax +# The following is the equivalent to -XX:+PrintGCDetails -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=10M +# -Xlog:gc=info,heap*=debug,age*=debug,safepoint=info,promotion*=debug:file=/var/log/cassandra/gc.log:time,uptime,pid,tid,level:filecount=10,filesize=10485760 + +# Notes for Java 8 migration: +# +# -XX:+PrintGCDetails maps to -Xlog:gc*:... - i.e. add a '*' after "gc" +# -XX:+PrintGCDateStamps maps to decorator 'time' +# +# -XX:+PrintHeapAtGC maps to 'heap' with level 'trace' +# -XX:+PrintTenuringDistribution maps to 'age' with level 'debug' +# -XX:+PrintGCApplicationStoppedTime maps to 'safepoint' with level 'info' +# -XX:+PrintPromotionFailure maps to 'promotion' with level 'trace' +# -XX:PrintFLSStatistics=1 maps to 'freelist' with level 'trace' + +### Netty Options + +# On Java >= 9 Netty requires the io.netty.tryReflectionSetAccessible system property to be set to true to enable +# creation of direct buffers using Unsafe. Without it, this falls back to ByteBuffer.allocateDirect which has +# inferior performance and risks exceeding MaxDirectMemory +-Dio.netty.tryReflectionSetAccessible=true + +### Enable vector incubator feature (simd support) + +--add-modules jdk.incubator.vector + +### Compatibility Options +--add-opens java.base/java.io=ALL-UNNAMED --add-opens java.base/sun.nio.ch=ALL-UNNAMED --add-opens java.base/java.nio=ALL-UNNAMED --add-opens java.base/java.lang=ALL-UNNAMED --add-opens java.base/java.util=ALL-UNNAMED +-Djava.security.manager=allow + +# The newline in the end of file is intentional \ No newline at end of file diff --git a/ide/idea/workspace.xml b/ide/idea/workspace.xml index 8337c5eee7b9..6e38eef56033 100644 --- a/ide/idea/workspace.xml +++ b/ide/idea/workspace.xml @@ -151,6 +151,7 @@ -Dcassandra.storagedir=$PROJECT_DIR$/data -Djava.library.path=$PROJECT_DIR$/lib/sigar-bin -Dlogback.configurationFile=file://$PROJECT_DIR$/conf/logback.xml + -Dnet.bytebuddy.experimental=true -XX:HeapDumpPath=build/test -ea" />