[WFCORE-6578] WildFly heap exhaustion via deserialization

wildfly · Nov 9, 2023 · 8baf794 · 8baf794
1 parent 408277f
commit 8baf794
Show file tree

Hide file tree

Showing 12 changed files with 107 additions and 2 deletions.
diff --git a/core-feature-pack/common/src/main/resources/content/bin/domain.bat b/core-feature-pack/common/src/main/resources/content/bin/domain.bat
@@ -120,6 +120,17 @@ set "HOST_CONTROLLER_JAVA_OPTS=!HOST_CONTROLLER_JAVA_OPTS! !SECURITY_MANAGER_CON
 
 setlocal DisableDelayedExpansion
 
+rem Add -Djdk.serialFilter if not specified
+echo "%JAVA_OPTS%" | findstr /I "\-Djdk.serialFilter" > nul
+if errorlevel == 1 (
+  if "x%DISABLE_JDK_SERIAL_FILTER%" == "x" (
+    setlocal EnableDelayedExpansion
+    set "PROCESS_CONTROLLER_JAVA_OPTS=!PROCESS_CONTROLLER_JAVA_OPTS! -Djdk.serialFilter="!JDK_SERIAL_FILTER!""
+    set "HOST_CONTROLLER_JAVA_OPTS=!HOST_CONTROLLER_JAVA_OPTS! -Djdk.serialFilter="!JDK_SERIAL_FILTER!""
+    setlocal DisableDelayedExpansion
+  )
+)
+
 rem Find run.jar, or we can't continue
 if exist "%JBOSS_HOME%\jboss-modules.jar" (
     set "RUNJAR=%JBOSS_HOME%\jboss-modules.jar"

diff --git a/core-feature-pack/common/src/main/resources/content/bin/domain.conf b/core-feature-pack/common/src/main/resources/content/bin/domain.conf
@@ -42,6 +42,16 @@ if [ "x$JBOSS_MODULES_SYSTEM_PKGS" = "x" ]; then
    JBOSS_MODULES_SYSTEM_PKGS="org.jboss.byteman"
 fi
 
+# Default JDK_SERIAL_FILTER settings
+#
+if [ "x$JDK_SERIAL_FILTER" = "x" ]; then
+   JDK_SERIAL_FILTER="maxbytes=10485760;maxdepth=128;maxarray=100000;maxrefs=300000"
+fi
+
+# Uncomment the following line to disable jdk.serialFilter settings
+#
+#DISABLE_JDK_SERIAL_FILTER=true
+
 #
 # Specify options to pass to the Java VM.
 #

diff --git a/core-feature-pack/common/src/main/resources/content/bin/domain.conf.bat b/core-feature-pack/common/src/main/resources/content/bin/domain.conf.bat
@@ -60,6 +60,14 @@ set "JAVA_OPTS=%JAVA_OPTS% -Djava.awt.headless=true"
 rem # Use JBoss Modules lockless mode
 rem set "JAVA_OPTS=%JAVA_OPTS% -Djboss.modules.lockless=true"
 
+rem # Default JDK_SERIAL_FILTER settings
+if "x%JDK_SERIAL_FILTER%" == "x" (
+  set "JDK_SERIAL_FILTER=maxbytes=10485760;maxdepth=128;maxarray=100000;maxrefs=300000"
+)
+
+rem # Uncomment the following line to disable jdk.serialFilter settings
+rem set "DISABLE_JDK_SERIAL_FILTER=true"
+
 :JAVA_OPTS_SET
 
 rem The ProcessController process uses its own set of java options

diff --git a/core-feature-pack/common/src/main/resources/content/bin/domain.conf.ps1 b/core-feature-pack/common/src/main/resources/content/bin/domain.conf.ps1
@@ -55,6 +55,16 @@ if (-Not $JAVA_OPTS) {
 
     # Use JBoss Modules lockless mode
     # $JAVA_OPTS += '-Djboss.modules.lockless=true'
+    # Default JDK_SERIAL_FILTER settings
+    #
+    if (-Not(test-path env:JDK_SERIAL_FILTER)) {
+        $JDK_SERIAL_FILTER = 'maxbytes=10485760;maxdepth=128;maxarray=100000;maxrefs=300000'
+    }
+
+    # Uncomment the following line to disable jdk.serialFilter settings
+    #
+    # $DISABLE_JDK_SERIAL_FILTER=$true
+
 }
 
 # Uncomment this to run with a security manager enabled

diff --git a/core-feature-pack/common/src/main/resources/content/bin/domain.ps1 b/core-feature-pack/common/src/main/resources/content/bin/domain.ps1
@@ -22,6 +22,13 @@ if ($SECMGR) {
     $MODULE_OPTS +="-secmgr";
 }
 
+$DISABLE_JDK_SERIAL_FILTER = Get-Env-Boolean DISABLE_JDK_SERIAL_FILTER $DISABLE_JDK_SERIAL_FILTER
+$JDK_SERIAL_FILTER = Get-Env JDK_SERIAL_FILTER $JDK_SERIAL_FILTER
+if (-Not($JAVA_OPTS -like "*-Djdk.serialFilter*") -and (-Not($DISABLE_JDK_SERIAL_FILTER))) {
+    $HOST_CONTROLLER_JAVA_OPTS += "-Djdk.serialFilter=$JDK_SERIAL_FILTER"
+    $PROCESS_CONTROLLER_JAVA_OPTS += "-Djdk.serialFilter=$JDK_SERIAL_FILTER"
+}
+
 Set-Global-Variables-Domain
 
 # consolidate the host-controller and command line opts
@@ -106,4 +113,4 @@ if ($SERVER_OPTS -ne $null){
 $backgroundProcess = Get-Env LAUNCH_JBOSS_IN_BACKGROUND 'false'
 $runInBackGround = $global:RUN_IN_BACKGROUND -or ($backgroundProcess -eq 'true')
 
-Start-WildFly-Process -programArguments $PROG_ARGS -runInBackground $runInBackGround
+Start-WildFly-Process -programArguments $PROG_ARGS -runInBackground $runInBackGround
diff --git a/core-feature-pack/common/src/main/resources/content/bin/domain.sh b/core-feature-pack/common/src/main/resources/content/bin/domain.sh
@@ -218,6 +218,13 @@ if [ "$SECMGR" = "true" ]; then
     MODULE_OPTS="$MODULE_OPTS -secmgr";
 fi
 
+# Check If jdk.serialFilter is specified
+JDK_FILTER_SET=`echo $JAVA_OPTS | $GREP "\-Djdk.serialFilter"`
+if [ "x$DISABLE_JDK_SERIAL_FILTER" = "x" -a "x$JDK_FILTER_SET" = "x" ]; then
+    PROCESS_CONTROLLER_JAVA_OPTS="$PROCESS_CONTROLLER_JAVA_OPTS -Djdk.serialFilter=\"$JDK_SERIAL_FILTER\""
+    HOST_CONTROLLER_JAVA_OPTS="$HOST_CONTROLLER_JAVA_OPTS -Djdk.serialFilter=\"$JDK_SERIAL_FILTER\""
+fi
+
 # Set default modular JVM options
 setDefaultModularJvmOptions $PROCESS_CONTROLLER_JAVA_OPTS
 PROCESS_CONTROLLER_JAVA_OPTS="$PROCESS_CONTROLLER_JAVA_OPTS $DEFAULT_MODULAR_JVM_OPTIONS"

diff --git a/core-feature-pack/common/src/main/resources/content/bin/standalone.bat b/core-feature-pack/common/src/main/resources/content/bin/standalone.bat
@@ -239,6 +239,18 @@ setlocal EnableDelayedExpansion
 call "!DIRNAME!common.bat" :setModularJdk
 setlocal DisableDelayedExpansion
 
+if not "%PRESERVE_JAVA_OPTS%" == "true" (
+  rem Add -Djdk.serialFilter if not specified
+  echo "%JAVA_OPTS%" | findstr /I "\-Djdk.serialFilter" > nul
+  if errorlevel == 1 (
+    if "x%DISABLE_JDK_SERIAL_FILTER%" == "x" (
+      setlocal EnableDelayedExpansion
+      set "JAVA_OPTS=!JAVA_OPTS! -Djdk.serialFilter="!JDK_SERIAL_FILTER!""
+      setlocal DisableDelayedExpansion
+    )
+  )
+)
+
 if not "%PRESERVE_JAVA_OPT%" == "true" (
     if "%GC_LOG%" == "true" (
         if not exist "%JBOSS_LOG_DIR%" > nul 2>&1 (

diff --git a/core-feature-pack/common/src/main/resources/content/bin/standalone.conf b/core-feature-pack/common/src/main/resources/content/bin/standalone.conf
@@ -46,6 +46,16 @@ fi
 #
 #PRESERVE_JAVA_OPTS=true
 
+# Default JDK_SERIAL_FILTER settings
+#
+if [ "x$JDK_SERIAL_FILTER" = "x" ]; then
+   JDK_SERIAL_FILTER="maxbytes=10485760;maxdepth=128;maxarray=100000;maxrefs=300000"
+fi
+
+# Uncomment the following line to disable jdk.serialFilter settings
+#
+#DISABLE_JDK_SERIAL_FILTER=true
+
 #
 # Specify options to pass to the Java VM.
 #

diff --git a/core-feature-pack/common/src/main/resources/content/bin/standalone.conf.bat b/core-feature-pack/common/src/main/resources/content/bin/standalone.conf.bat
@@ -86,6 +86,14 @@ rem set "GC_LOG=true"
 rem # Uncomment and edit to use a custom java.security file to override all the Java security properties
 rem set "JAVA_OPTS=%JAVA_OPTS% -Djava.security.properties==C:\path\to\custom\java.security"
 
+rem # Default JDK_SERIAL_FILTER settings
+if "x%JDK_SERIAL_FILTER%" == "x" (
+  set "JDK_SERIAL_FILTER=maxbytes=10485760;maxdepth=128;maxarray=100000;maxrefs=300000"
+)
+
+rem # Uncomment the following line to disable jdk.serialFilter settings
+rem set "DISABLE_JDK_SERIAL_FILTER=true"
+
 :JAVA_OPTS_SET
 
 rem # Uncomment to add a Java agent. If an agent is added to the module options, then jboss-modules.jar is added as an agent

diff --git a/core-feature-pack/common/src/main/resources/content/bin/standalone.conf.ps1 b/core-feature-pack/common/src/main/resources/content/bin/standalone.conf.ps1
@@ -76,6 +76,16 @@ if (-Not $JAVA_OPTS) {
 
     # Uncomment and edit to use a custom java.security file to override all the Java security properties
     # $JAVA_OPTS += '-Djava.security.properties==C:\path\to\custom\java.security'
+
+    # Default JDK_SERIAL_FILTER settings
+    #
+    if (-Not(test-path env:JDK_SERIAL_FILTER)) {
+        $JDK_SERIAL_FILTER = 'maxbytes=10485760;maxdepth=128;maxarray=100000;maxrefs=300000'
+    }
+
+    # Uncomment the following line to disable jdk.serialFilter settings
+    #
+    # $DISABLE_JDK_SERIAL_FILTER=$true
 }
 
 # Uncomment this to run with a security manager enabled

diff --git a/core-feature-pack/common/src/main/resources/content/bin/standalone.ps1 b/core-feature-pack/common/src/main/resources/content/bin/standalone.ps1
@@ -37,11 +37,18 @@ if ($global:DEBUG_MODE){
     }
 }
 
+$DISABLE_JDK_SERIAL_FILTER = Get-Env-Boolean DISABLE_JDK_SERIAL_FILTER $DISABLE_JDK_SERIAL_FILTER
+$JDK_SERIAL_FILTER = Get-Env JDK_SERIAL_FILTER $JDK_SERIAL_FILTER
+if ($PRESERVE_JAVA_OPTS -ne 'true') {
+    if (-Not($JAVA_OPTS -like "*-Djdk.serialFilter*") -and (-Not($DISABLE_JDK_SERIAL_FILTER))) {
+        $JAVA_OPTS += "-Djdk.serialFilter=$JDK_SERIAL_FILTER"
+    }
+}
 $backgroundProcess = Get-Env LAUNCH_JBOSS_IN_BACKGROUND 'false'
 $runInBackGround = $global:RUN_IN_BACKGROUND -or ($backgroundProcess -eq 'true')
 
 $PROG_ARGS = Get-Java-Arguments -entryModule "org.jboss.as.standalone" -serverOpts $SERVER_OPTS
 
 Display-Environment $global:FINAL_JAVA_OPTS
 
-Start-WildFly-Process -programArguments $PROG_ARGS -runInBackground $runInBackGround
+Start-WildFly-Process -programArguments $PROG_ARGS -runInBackground $runInBackGround
diff --git a/core-feature-pack/common/src/main/resources/content/bin/standalone.sh b/core-feature-pack/common/src/main/resources/content/bin/standalone.sh
@@ -279,6 +279,11 @@ if [ "$PRESERVE_JAVA_OPTS" != "true" ]; then
         fi
     fi
 
+    JDK_FILTER_SET=`echo $JAVA_OPTS | $GREP "\-Djdk.serialFilter"`
+    if [ "x$DISABLE_JDK_SERIAL_FILTER" = "x" -a "x$JDK_FILTER_SET" = "x" ]; then
+        PREPEND_JAVA_OPTS="$PREPEND_JAVA_OPTS -Djdk.serialFilter=\"$JDK_SERIAL_FILTER\""
+    fi
+
     # Set default modular JVM options
     setDefaultModularJvmOptions $JAVA_OPTS
     JAVA_OPTS="$JAVA_OPTS $DEFAULT_MODULAR_JVM_OPTIONS"